{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先 import 必要的模块\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "import lightgbm as lgbm\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 87018 entries, 0 to 87017\n",
      "Data columns (total 81 columns):\n",
      "Gender_Female              87018 non-null float64\n",
      "Gender_Male                87018 non-null float64\n",
      "Mobile_Verified_N          87018 non-null float64\n",
      "Mobile_Verified_Y          87018 non-null float64\n",
      "Var1_HAVC                  87018 non-null float64\n",
      "Var1_HAXA                  87018 non-null float64\n",
      "Var1_HAXB                  87018 non-null float64\n",
      "Var1_HAXC                  87018 non-null float64\n",
      "Var1_HAXF                  87018 non-null float64\n",
      "Var1_HAXM                  87018 non-null float64\n",
      "Var1_HAYT                  87018 non-null float64\n",
      "Var1_HAZD                  87018 non-null float64\n",
      "Var1_HBXA                  87018 non-null float64\n",
      "Var1_HBXB                  87018 non-null float64\n",
      "Var1_HBXC                  87018 non-null float64\n",
      "Var1_HBXD                  87018 non-null float64\n",
      "Var1_HBXH                  87018 non-null float64\n",
      "Var1_HBXX                  87018 non-null float64\n",
      "Var1_HCXD                  87018 non-null float64\n",
      "Var1_HCXF                  87018 non-null float64\n",
      "Var1_HCXG                  87018 non-null float64\n",
      "Var1_HCYS                  87018 non-null float64\n",
      "Var1_HVYS                  87018 non-null float64\n",
      "Filled_Form_N              87018 non-null float64\n",
      "Filled_Form_Y              87018 non-null float64\n",
      "Device_Type_Mobile         87018 non-null float64\n",
      "Device_Type_Web-browser    87018 non-null float64\n",
      "Var2_A                     87018 non-null float64\n",
      "Var2_B                     87018 non-null float64\n",
      "Var2_C                     87018 non-null float64\n",
      "Var2_D                     87018 non-null float64\n",
      "Var2_E                     87018 non-null float64\n",
      "Var2_F                     87018 non-null float64\n",
      "Var2_G                     87018 non-null float64\n",
      "Source_S122                87018 non-null float64\n",
      "Source_S123                87018 non-null float64\n",
      "Source_S124                87018 non-null float64\n",
      "Source_S125                87018 non-null float64\n",
      "Source_S127                87018 non-null float64\n",
      "Source_S129                87018 non-null float64\n",
      "Source_S130                87018 non-null float64\n",
      "Source_S133                87018 non-null float64\n",
      "Source_S134                87018 non-null float64\n",
      "Source_S135                87018 non-null float64\n",
      "Source_S136                87018 non-null float64\n",
      "Source_S137                87018 non-null float64\n",
      "Source_S138                87018 non-null float64\n",
      "Source_S139                87018 non-null float64\n",
      "Source_S140                87018 non-null float64\n",
      "Source_S141                87018 non-null float64\n",
      "Source_S143                87018 non-null float64\n",
      "Source_S144                87018 non-null float64\n",
      "Source_S150                87018 non-null float64\n",
      "Source_S151                87018 non-null float64\n",
      "Source_S153                87018 non-null float64\n",
      "Source_S154                87018 non-null float64\n",
      "Source_S155                87018 non-null float64\n",
      "Source_S156                87018 non-null float64\n",
      "Source_S157                87018 non-null float64\n",
      "Source_S158                87018 non-null float64\n",
      "Source_S159                87018 non-null float64\n",
      "Source_S160                87018 non-null float64\n",
      "Source_S161                87018 non-null float64\n",
      "Source_S162                87018 non-null float64\n",
      "Monthly_Income             87018 non-null float64\n",
      "DOB_Year                   87018 non-null float64\n",
      "DOB_Month                  87018 non-null float64\n",
      "DOB_Day                    87018 non-null float64\n",
      "Lead_Creation_Year         87018 non-null int64\n",
      "Lead_Creation_Month        87018 non-null float64\n",
      "Lead_Creation_Day          87018 non-null float64\n",
      "Loan_Amount_Applied        87018 non-null float64\n",
      "Loan_Tenure_Applied        87018 non-null float64\n",
      "Existing_EMI               87018 non-null float64\n",
      "Var5                       87018 non-null float64\n",
      "Loan_Amount_Submitted      87018 non-null float64\n",
      "Loan_Tenure_Submitted      87018 non-null float64\n",
      "Interest_Rate              87018 non-null float64\n",
      "Processing_Fee             87018 non-null float64\n",
      "EMI_Loan_Submitted         87018 non-null float64\n",
      "Var4                       87018 non-null float64\n",
      "dtypes: float64(80), int64(1)\n",
      "memory usage: 53.8 MB\n"
     ]
    }
   ],
   "source": [
    "dpath = './'\n",
    "\n",
    "X_train = pd.read_csv(dpath +\"FE_Train2.csv\")\n",
    "y_train = X_train['Disbursed'] \n",
    "X_train = X_train.drop(['Disbursed'], axis=1)\n",
    "\n",
    "X_train.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "Name: Disbursed, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存特征名字以备后用（可视化）\n",
    "feat_names = X_train.columns \n",
    "\n",
    "#sklearn的学习器大多之一稀疏数据输入，模型训练会快很多\n",
    "#查看一个学习器是否支持稀疏数据，可以看fit函数是否支持: X: {array-like, sparse matrix}.\n",
    "#可自行用timeit比较稠密数据和稀疏数据的训练时间\n",
    "#from scipy.sparse import csr_matrix\n",
    "#X_train = csr_matrix(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "LightGBM超参数调优\n",
    "LightGBM的主要的超参包括：\n",
    "\n",
    "树的数目n_estimators 和 学习率 learning_rate\n",
    "树的最大深度max_depth 和 树的最大叶子节点数目num_leaves（注意：XGBoost只有max_depth，LightGBM采用叶子优先的方式生成树，num_leaves很重要，设置成比 2^max_depth 小）\n",
    "叶子结点的最小样本数:min_data_in_leaf(min_data, min_child_samples)\n",
    "每棵树的列采样比例：feature_fraction/colsample_bytree\n",
    "每棵树的行采样比例：bagging_fraction （需同时设置bagging_freq=1）/subsample\n",
    "正则化参数lambda_l1(reg_alpha), lambda_l2(reg_lambda)\n",
    "\n",
    "两个非模型复杂度参数，但会影响模型速度和精度。可根据特征取值范围和样本数目修改这两个参数 1）特征的最大bin数目max_bin：默认255； 2）用来建立直方图的样本数目subsample_for_bin：默认200000。\n",
    "\n",
    "对n_estimators，用LightGBM内嵌的cv函数调优，因为同XGBoost一样，LightGBM学习的过程内嵌了cv，速度极快。 其他参数用GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_ROUNDS = 10000"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "相同的交叉验证分组"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=3, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "默认参数，此时学习率为0.1，比较大，观察弱分类数目的大致范围 （采用默认参数配置，看看模型是过拟合还是欠拟合）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_array = np.array(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "def modelfit(alg, X_train_value, y_train_value, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()    \n",
    "    #直接调用xgboost，而非sklarn的wrapper类\n",
    "    xgtrain = xgb.DMatrix(X_train_value, label = y_train_value)\n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='logloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    #最佳参数n_estimators\n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "    # 采用交叉验证得到的最佳参数n_estimators，训练模型\n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train_value, y_train_value, eval_metric='logloss')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\xgboost\\core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n",
      "D:\\Anaconda3\\lib\\site-packages\\xgboost\\core.py:588: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  data.base is not None and isinstance(data, np.ndarray) \\\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators,时间关系 n_estimators 以及depth都比较小\n",
    "        max_depth=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train_array, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#n_estimators调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3XmcHXWd7//X+5zekk5n75ClQxKyQcJOAy6gKOiASwAHERQV1+uMXPdxQOfnb4a5OONyXWZER1RARUUGHURlRAXEhREJa0hCIGTfO/ve6+f+UdXNoelOd0Kfru4+7+fjUY8+VfWtqs+pc/q8Ty2nShGBmZkZQC7rAszMbOBwKJiZWQeHgpmZdXAomJlZB4eCmZl1cCiYmVkHh4JZAUmfkvTtrOswy4pDYZCRNELSKklvLRhWI2mNpEsKhtVL+oWkHZJ2Sloi6TpJY9LxV0pqlbQ37VZI+psi136OpHXFXMbh6KqeiPhsRLy3SMtbJem8Ysy7GPrr9Rps62WocygMMhGxF3g/8FVJtengzwMLI+J2AEkvA34H/Ak4NiJGA+cDLcBJBbP7n4gYEREjgEuAz0s6pX+eiR0OSWVZ12AlIiLcDcIOuBn4EXAOsA2YVDDuj8C/9zD9lcAfOw37C/DWgv4FwGJgJ0nIHFcw7rh02M60zYKCca8DlgB7gPXAJ4Bq4ADQBuxNu8ndPK/rgV+m0z8IzOzF+jgW+A2wHVgGXHok9QD/CNySTjcdCOBdwFpgB/AB4HTgifS5f61gOTOBe9PXYyvwA2B0Ou776bIOpMv6ZC/W8Srg79NlNQJlaf/69LksA87tYl28BNgE5AuGXQw8kT4+A1gI7AY2A1/qZp2eA6zrZtwo4HtAA7Aa+Acgl47LA/83XQcrgavS9VjWzbxWAed1M+59wPL0db2z/T0DCPgysAXYla6j47t7vbP+fx1MXeYFuDvCFw7GABvTf7x3FQyvBlqBc3qY/koKQiH9oNsJzEn75wD7gNcA5cAn03/OirR/OfCptP/V6T/g3HTajcDZBXWemj7u9kOmoI6b0w+AM9IPwR8At/YwTTXJh/a70mlOTdfL/MOth65D4T+AKuC1wEHgDmACMCX9UHpl2n5Wur4qgVrg98BXCub9vA+/Q63jgvaPAVOBYcDc9HlOLqivy8AEngVeU9D/n8DV6eP/Ad6ePh4BvKSbeXT7epEEws+AmrSOp4H3pOM+QPKhXJeu799yBKGQvq+2pq9nJfDvwO/TcX8FPAyMJgmI40i/GHX3ervrXefdR4NUROwg+YY5HPhpwagxJLsFN7UPkPT59LjCPkn/UND2JenwvSRbCd8HnknHvQX4ZUT8JiKagS+SfDC9jOSb6AjgXyOiKSLuBX4BXJ5O2wzMkzQyInZExCOH+fR+GhF/iYgWklA4uYf2bwBWRcRNEdGSLu8nJLvE+qKef46IgxHxa5IP8R9FxJaIWA/8ATgFICKWp+urMSIagC8BrzzEfA+1jtv9W0SsjYgDJGFfmT6X8ohYFRHPdjPvH5G+HpJqSL49/6hgfcySND4i9kbEnw9nZUjKp7VfExF7ImIVyZbB29MmlwJfjYh16fv0Xw9n/gXeBtwYEY9ERCNwDfBSSdPT51BDsoWoiFgaERsLnt+Leb1LmkNhkJJ0Bck3tN8CnysYtYNkN8Wk9gER8clIjiv8F8k36XZ/jojRkRxTmAjMBz6bjptMslugfR5tJN9Sp6Tj1qbD2q1OxwH8NcmH0GpJ90t66WE+vU0Fj/eTBNChTAPOTANup6SdJB8oE/uons0Fjw900T8CQNIESbdKWi9pN3ALMP4Q8z3UOm63tmD8cuAjJFszW9JlTe5m3j8E3iSpEngT8EhEtC/rPSRbKU9JekjSGw5RY1fGk2whri4YVvj6Ty6su9Pjw9F5/ewl2TU3Jf0i8jWSXY2bJd0gaWTa9MW+3iXNoTAISZpAsj/1fcD/Ai6V9AqAiNhHsh/+TYczz4jYTPLt+o3poA0kH7btyxTJboz16bipkgrfP0en44iIhyLiQpJdLHcAt7Uv5nBqOgxrgfvTgGvvRkTE3/RzPf+SzvPEiBgJXEGya6Nd5+Udah13OU1E/DAizkqnC57/haCw3RKSD9QLgLeShET7uGci4nKS9fE54HZJ1b1/mmwl+TY+rWBYx+tPsvumrmDc1MOYd6HO66caGMdz77N/i4jTSL7MzAH+Lh3e3ettveBQGJy+BtwREfelm8yfBL6Vfisk7X+3pKvTAEFSHTCjuxlKGkdyMHJxOug24PWSzpVUDnyc5GDnAyShsw/4pKRySeeQhMmtkiokvU3SqHSXyG6S3R6QfMMeJ2lUH62Hdr8A5kh6e1pPuaTTJR3Xz/XUkBxE3ilpCumHVIHNwDEF/Ydaxy8gaa6kV6ev80GSrZTWrtqmfgh8CHgFyTGF9vlcIak23TLZmQ7udj6Sqgo7ki3R24Dr0tOhpwEfI9kyan9eH5Y0RdJokoPjPSnvtJyytP53STo5fc6fBR6MiFXp63tmut72peujtYfX23oj64Ma7g6vAy4i+QY1utPwe4DrCvrPBO4i+affCTwJXAeMS8dfSfLP0n7mzRaSfc4TCuZxMckBw13A/aQHbtNx89Nhu9I2F6fDK4BfkezG2g08BJxVMN2NJLsAdtL92Uf/p6D/HHo4OJ22m0tyxlJDOv97SY5FHFY9dH2guayg/ToKDuKTfBD+Q8E6eThdn4+RfMivK2h7IbAmXdYnerGOV/H8A9Mnkhz72UNyMP4XXa3DgvZHk3yA/7LT8FvS13svyZeAi7qZ/pz0+XfuZpEcu7olXd9rgc/w3NlHZSRbsttIzj76KMmWhbpZzqoulvF/0nEfIDlo3v5869Lh55KccbSX5870GtHT6+2u507pCjYzKwpJFwD/ERHTemxsmfPuIzPrU5KGSXqdpLJ0N9r/T3KSgw0C3lKwQUHS2cB/dzUukrOnbICQNJxkV9ixJMc9fgl8OCJ2Z1qY9YpDwczMOnj3kZmZdRh0F9kaP358TJ8+PesyzMwGlYcffnhrRNT21G7QhcL06dNZuHBh1mWYmQ0qklb33Mq7j8zMrIBDwczMOjgUzMysg0PBzMw6OBTMzKyDQ8HMzDo4FMzMrENRQ0HS+ZKWSVou6epu2lwqaYmkxZJ+2FWbvrBj0yqe+dNPe25oZlbCivbjtfQ+rteT3JR8HfCQpDsjuSNUe5vZJPddfXlE7Gi/IUwxLL3xb3hZ0wPsO+FVVI8cU6zFmJkNasXcUjgDWB4RKyKiCbiV5CYjhd4HXB/Jzb2JiC3FKqbipOQe7g1rny7WIszMBr1ihsIUnn/D7nU8/4bkkNxXdY6kP0n6s6Tzu5qRpPdLWihpYUNDwxEVUzNxJgC7Niw/ounNzEpBMUNBXQzrfJ3uMmA2yW3/Lge+nd7T9fkTRdwQEfURUV9b2+P1nLo0fuocAA40rDii6c3MSkExQ2EdMLWgv47k3sKd2/wsIpojYiWwjCQk+tzY8RPZF1VoZ6+uCWVmVpKKGQoPAbMlzZBUAVwG3NmpzR3AqwAkjSfZnVSUr/LK5dicn0jVnrU9NzYzK1FFC4WIaAGuAu4GlgK3RcRiSddKWpA2uxvYJmkJcB/wdxGxrVg17aqaxKjGzhsrZmbWrqj3U4iIu4C7Og37TMHjAD6WdkXXWD2VCfseIdraUM6/2zMz66y0PhnHTGO4Gtm5dWPWlZiZDUglFQqVtTMA2OrfKpiZdamkQmHUpFkA7N7k3yqYmXWlpEJhwrS5ADRvXZlxJWZmA1NJhcKImtFsZyS5XWuyLsXMbEAqqVAAaCibyPB967Iuw8xsQCq5UNgzbApjmnz2kZlZV0ouFJpHTGVCWwOtLS1Zl2JmNuCUXCjkxk6nXK1s3eAL45mZdVZyoTBswjEAbFv3TMaVmJkNPCUXCmOmJBdh3bf52YwrMTMbeEouFCZMnUlbiNZtq7IuxcxswCm5UKisHMYWjaNst3+rYGbWWcmFAsD28kmMOLA+6zLMzAackgyFvcOnMLZ5U9ZlmJkNOCUZCq0jj2YC22k8uC/rUszMBpSSDIWycdMB2LLGp6WamRUqyVConjgTgB3rfQltM7NCJRkK4+qS3yocaPCvms3MCpVkKNROms7BKCe2ORTMzAqVZCjk8nk25KcwbLdDwcysUEmGAsDO4dMYf3B11mWYmQ0oJRsKTaNnMbFtM40H92ddipnZgFGyoVB21FzyCjauXJJ1KWZmA0bJhsKoqfMB2L76yYwrMTMbOEo2FCbPPB6Apk3LMq7EzGzgKNlQqK4ZzSbGU7bdv2o2M2tX1FCQdL6kZZKWS7q6i/FXSmqQ9FjavbeY9XTWUDmVkft9BpKZWbuihYKkPHA9cAEwD7hc0rwumv44Ik5Ou28Xq56u7K85hsnNa4m2tv5crJnZgFXMLYUzgOURsSIimoBbgQuLuLzDFuPnMEIH2LbJN9wxM4PihsIUYG1B/7p0WGd/LekJSbdLmtrVjCS9X9JCSQsbGhr6rMDqKccBsGnFE302TzOzwayYoaAuhkWn/p8D0yPiROC3wHe7mlFE3BAR9RFRX1tb22cFTjjmBAD2bXiqz+ZpZjaYFTMU1gGF3/zrgA2FDSJiW0Q0pr3fAk4rYj0vMGHSdPZFFdHwdH8u1sxswCpmKDwEzJY0Q1IFcBlwZ2EDSZMKehcAS4tYzwsol2NDWR3Ve3xhPDMzgLJizTgiWiRdBdwN5IEbI2KxpGuBhRFxJ/AhSQuAFmA7cGWx6unOrurp1O1+vL8Xa2Y2IBUtFAAi4i7grk7DPlPw+BrgmmLW0JPmMbOYuPu3HNi3h2HVNVmWYmaWuZL9RXO7ionHArDh2UUZV2Jmlr2SD4Ux05IL4+1YuzjjSszMslfyoTB5xnzaQrT4wnhmZg6FqmHVbMxNoHzns1mXYmaWuZIPBYCtVdMYvX9V1mWYmWXOoQAcaG5jcss6Wltasi7FzCxTDgUgN/8ihqmJ9St8FzYzK20OBWDsMacC0LD84YwrMTPLlkMBmDr3FJojT9N6Xy3VzEqbQwGorBrO2vxUhm9fknUpZmaZciikto2YzaQDy7Muw8wsUw6FVOuE+UxgOzu3bsq6FDOzzDgUUtVHnwzAuqf+knElZmbZcSikJs89HYC9qx/LuBIzs+w4FFLjjqpjK6PJb/GF8cysdDkUCmyomsXYvb41p5mVLodCgX1jjmVqyxqamxp7bmxmNgQ5FAqUTz6RCrWw7hkfVzCz0uRQKDB+1mkAbF3+SMaVmJllw6FQoG7WiTRGOa0bHs+6FDOzTDgUCpSVV7C27Giqdz6VdSlmZplwKHSyvWYuUxqfJdrasi7FzKzfORQ6iaPmM5bdbNu0NutSzMz6nUOhk5pppwCwYdlDGVdiZtb/HAqdTDnuDAD2r/YZSGZWehwKnYwaU8taTaZyy6NZl2Jm1u8cCl3YNPJ4pu5f4oPNZlZyHApdaJt8GuPZycY1z2RdiplZvypqKEg6X9IyScslXX2IdpdICkn1xaynt8bNeRkAGxb/MeNKzMz6V9FCQVIeuB64AJgHXC5pXhftaoAPAQ8Wq5bDNW3+GTRGOc1rfAaSmZWWYm4pnAEsj4gVEdEE3Apc2EW7fwY+DxwsYi2HpbyiipUVsxmz3Ze7MLPSUsxQmAIU/gJsXTqsg6RTgKkR8YtDzUjS+yUtlLSwoaGh7yvtws6xJzK96RmaGn0ZbTMrHcUMBXUxLDpGSjngy8DHe5pRRNwQEfURUV9bW9uHJXavfNrpVKmZ1Ut9z2YzKx3FDIV1wNSC/jpgQ0F/DXA88DtJq4CXAHcOlIPNk+efDcD2ZQ9kXImZWf8pZig8BMyWNENSBXAZcGf7yIjYFRHjI2J6REwH/gwsiIiFRayp1yZOnc02RpHb8HDWpZiZ9ZseQ0HSTEmV6eNzJH1I0uiepouIFuAq4G5gKXBbRCyWdK2kBS+28GJTLsfa4fM5aveTWZdiZtZverOl8BOgVdIs4DvADOCHvZl5RNwVEXMiYmZEXJcO+0xE3NlF23MGylZCu4MTTuHoWM+u7VuyLsXMrF/0JhTa0m/9FwNfiYiPApOKW9bAUDPzTABWPfGHjCsxM+sfvQmFZkmXA+8E2k8dLS9eSQPHtJPOpi3E/hV/zroUM7N+0ZtQeBfwUuC6iFgpaQZwS3HLGhhGjBzLmvxUhm15LOtSzMz6RVlPDSJiCcllKJA0BqiJiH8tdmEDRcOoE5iz43e0traSz+ezLsfMrKh6c/bR7ySNlDQWeBy4SdKXil/awKAZZzGKfaxcPGAuzWRmVjS92X00KiJ2A28CboqI04DzilvWwDH11NcC0LDonowrMTMrvt6EQpmkScClPHeguWQcVTeL9ZpI1fo/ZV2KmVnR9SYUriX5AdqzEfGQpGOAkrr7zPox9czc/wStLS1Zl2JmVlQ9hkJE/GdEnBgRf5P2r4iIvy5+aQNHfsbZjGQfKxf71FQzG9p6c6C5TtJ/SdoiabOkn0iq64/iBoqjTzsfgK0+rmBmQ1xvdh/dRHIhu8kk90P4eTqsZNROns5aTaZqva+YamZDW29CoTYiboqIlrS7GeifmxoMIBvH1DNz/+M+rmBmQ1pvQmGrpCsk5dPuCmBbsQsbaHIzX0GNDrBikbcWzGzo6k0ovJvkdNRNwEbgEpJLX5SU6ackv1fY9uRvM67EzKx4enP20ZqIWBARtRExISIuIvkhW0kZP3kaq3N1DNvwP1mXYmZWNEd657WP9WkVg8SmMfXM3L+IluamrEsxMyuKIw0F9WkVg0TZzFcwQgdY8YR/3WxmQ9ORhkL0aRWDxIz682kLsf2JX2VdiplZUXR76WxJe+j6w1/AsKJVNICNnTCFZeVzGLP+vqxLMTMrim63FCKiJiJGdtHVRESP92EYqrZPfiWzm59m+5b1WZdiZtbnjnT3Uckaf+oCcgpW/PlnWZdiZtbnHAqHaeYJL2Mro9Ezv8m6FDOzPudQOEy5fJ4Vo1/G7D0P+tRUMxtyHApHoGzuaxnJPp555N6sSzEz61O9uXT2Hkm7O3Vr08tpH9MfRQ40s166gObIs/PxX2ZdiplZn+rNWURfAjYAPyQ5HfUyYCKwDLgROKdYxQ1UI0ePY3HlfCZu+n3WpZiZ9ane7D46PyK+GRF7ImJ3RNwAvC4ifgyMKXJ9A9aeo1/NjLZVbFq7POtSzMz6TG9CoU3SpZJyaXdpwbhD/rJZ0vmSlklaLunqLsZ/QNIiSY9J+qOkeYf7BLIy6bQFAKz+8x0ZV2Jm1nd6EwpvA94ObEm7twNXSBoGXNXdRJLywPXABcA84PIuPvR/GBEnRMTJwOdJdlUNCkfPPYUNmkDlCp+aamZDR4/HFCJiBfDGbkb/8RCTngEsT6dH0q3AhcCSgnnvLmhfzSC6ppJyOdaMfyWnbLmDPbu2UzNqbNYlmZm9aL05+6guPdNoi6TNkn4iqa4X854CrC3oX5cO6zz/D0p6lmRL4UPd1PB+SQslLWxoaOjFovvH6PpLqVQzy+6/LetSzMz6RG92H90E3AlMJvlQ/3k6rCddXV77BVsCEXF9RMwE/h74h65mFBE3RER9RNTX1g6c20PPqT+XLYwl/5QveWFmQ0NvQqE2Im6KiJa0uxnozSfzOmBqQX8dyamt3bkVuKgX8x0wcvk8Kya8hvn7/sLunSV322ozG4J6EwpbJV0hKZ92VwC9+QR8CJgtaYakCpLfN9xZ2EDS7ILe1wPP9LbwgWLMGZdSoRaW3f/jrEsxM3vRehMK7wYuBTYBG4FLgHf1NFFEtJCcnXQ3sBS4LSIWS7pW0oK02VWSFkt6jOQWn+88gueQqTmnvopNjKfcu5DMbAjozdlHa4AFhcMkfQT4Si+mvQu4q9OwzxQ8/nCvKx2glMuz6qjXcOqm29i1fSujxo7PuiQzsyN2pBfE+1ifVjHIjT3zMirUyrL7b826FDOzF+VIQ6GrM4tK1uyTX8FGaql82ruQzGxwO9JQGDQ/MusPyuVYPfG1zNv/MLu2bc66HDOzI9ZtKHRzyezdkvaQ/GbBCox/yeWUq5Wl93wv61LMzI5Yt6EQETURMbKLriYienPJ7ZIy88SXszI3nbFP+9fNZjZ4+c5rfUS5HJtnXcKclqdZ8eSDWZdjZnZEHAp96NjXvJemyLP599/JuhQzsyPiUOhDo2sn8WTNWczdcheNjQeyLsfM7LA5FPpYef07GMseFt3ry16Y2eDjUOhj8866KLly6uM/zLoUM7PD5lDoY/myMlbWXciJB/7CxnUrsy7HzOywOBSK4OhXv4+8ghW/+VbWpZiZHRaHQhFMOmY+SypP4pjVt9Hc3JR1OWZmveZQKJKW09/HJBp49Nc/yLoUM7NecygUyfHnXM4GHcWIR28gwpeKMrPBwaFQJLmyMtbPeQfzWpbw1MO/y7ocM7NecSgU0bzX/y17Gcbe+/8961LMzHrFoVBE1SPHsmTihZy8+3dsWPts1uWYmfXIoVBk0y74KDnaWHnXV7MuxcysRw6FIjtq2rEsqjmL+Rt/wt49u7Iux8zskBwK/aD6nA8zmr08fseXsi7FzOyQHAr9YHb9a1hcdQrHPnuTtxbMbEBzKPSTinM/zTh28fh//d+sSzEz65ZDoZ/MPv01LK46leNWeGvBzAYuh0I/Kj/v04xlN094a8HMBiiHQj+aU38eT1adxrErbmTP7h1Zl2Nm9gIOhX5Wed6nkzuz/fSLWZdiZvYCDoV+Nrv+XJ4YdiYnrPwODZvXZV2OmdnzFDUUJJ0vaZmk5ZKu7mL8xyQtkfSEpHskTStmPQPF2Is/zzAaWf7jT2ddipnZ8xQtFCTlgeuBC4B5wOWS5nVq9ihQHxEnArcDny9WPQNJ3ZyTefSoN3HGtp+x/Mm/ZF2OmVmHYm4pnAEsj4gVEdEE3ApcWNggIu6LiP1p75+BuiLWM6DMveyz7NNw9t3590RbW9blmJkBxQ2FKcDagv516bDuvAf4765GSHq/pIWSFjY0NPRhidkZOfYolh37QU5qeoRH7rkt63LMzIDihoK6GNblLcgkXQHUA1/oanxE3BAR9RFRX1tb24clZuuUN32CtbkpjH/gWg4ePJB1OWZmRQ2FdcDUgv46YEPnRpLOAz4NLIiIxiLWM+CUVVSy+xXXMi3Ws/CW/y/rcszMihoKDwGzJc2QVAFcBtxZ2EDSKcA3SQJhSxFrGbDmn3MJj4w6jzPW3sgzi3zQ2cyyVbRQiIgW4CrgbmApcFtELJZ0raQFabMvACOA/5T0mKQ7u5ndkDbr7V9jn6ppu+ODNDc3Z12OmZUwRXS5m3/Aqq+vj4ULF2ZdRp97/K5vc9JfPs4fjvkoZ7/jH7Mux8yGGEkPR0R9T+38i+YB4qQL3sOi6pdS/+z1rHp6UdblmFmJcigMFBKT3/YNWpRn/23vo7GppI65m9kA4VAYQMZNnsGql1zHvJalPHjj32VdjpmVIIfCAHPC+e/h0XGv56yN32PhfXdkXY6ZlRiHwgA07z3/wfr8FI6+/yNs3ri25wnMzPqIQ2EAqhw+Er35RkbFXtbf9C6aW1qyLsnMSoRDYYCqO+5Mlp50Dac2PcQDN3yYwXbqsJkNTg6FAezkiz/GoxMu4pVbbuGPP/la1uWYWQlwKAxkEie+71s8VXUSZyz6Rx77n19nXZGZDXEOhQEuX15B3f+6nW35Wurufi+rnl2WdUlmNoQ5FAaBEWMmoLfeSiXN6JaL2bjBZySZWXE4FAaJSbNOZusbv8uEaGDvtxewdevQuNmQmQ0sDoVBZMZpr2Xda25gWutqNv3HAnbt2pV1SWY2xDgUBpnZL7+YZ8/+Msc1L2XF9Rexa/furEsysyHEoTAIHXfeO1l6+nWc1Pgoa/7tArZt25p1SWY2RDgUBqnj3/BBlr38SxzbvJSG6/+KjRvXZV2SmQ0BDoVB7LjXvptV532TGW2rOXDD+axc4dNVzezFcSgMcrPPfjMb3nALE2IrI777Wh5+4DdZl2Rmg5hDYQiYUX8++664i9Z8JfPvvpz7bv+Gr5VkZkfEoTBEHDXrVGr+9+9ZM+xYXvXk1dz79f/NgYO+e5uZHR6HwhBSPWYisz5+D09MuJBzG77PM194FSt8WQwzOwwOhSEmV17JiX/7PZa+9AvMan2WMd87l9///PvenWRmveJQGKKO+6v3c+Bd97KrYgKvePgqfvJPl7B5iy+NYWaH5lAYwsZNm8/Uv/sTjx/9Di6Oe4ivn8n9v/iBtxrMrFsOhSEuXzGMk97972x+889pLqvmlQv/lj9+/mKWP/t01qWZ2QDkUCgRk48/mymffIhFM9/PmQf+wMTvnc2vb7iaHbv2ZF2amQ0gDoUSkquo4oS3f4ED732AtaPree2Gb7D7y/X85ravc6CxOevyzGwAKGooSDpf0jJJyyVd3cX4V0h6RFKLpEuKWYs9Z1TdXI776C9Z8/pbyJVX8Zol17D2X07j3jtuprG5JevyzCxDRQsFSXngeuACYB5wuaR5nZqtAa4EflisOqx7R5/+RqZe8yjLz/4qI/ItvPqxD7Pys6fz29u+zp79B7Iuz8wyUMwthTOA5RGxIiKagFuBCwsbRMSqiHgCaCtiHXYouRyzzr2SSZ96nKdf8jlG5ps4b8k17PzcSdx98z+zbpNPYzUrJcUMhSlA4c2E16XDDpuk90taKGlhQ4M/pIpB+XLmnP8BJn9qEavO+yYtw8byV6u+yKhvnMA9X7yCBx64n5ZWZ7fZUFdWxHmri2FHdIJ8RNwA3ABQX1/vk+yLKZdj+lmXwcvfQsPSP9Jw39c5q+FXVP765zz569msnfpGpr3yHRw3czpSVy+xmQ1mxQyFdcDUgv46YEMRl2d9SaJ23tnUzjublj1beeq332L00ts4fu2XaP7+V/lz2ansmH4B0152CfOOOdoBYTZEqFi/bpVUBjwNnAusBx4C3hoRi7toezPwi4i4vaf51tfXx8KFC/u4WuutPaseY93vb6Z29S8Y39pAc+R5NH88Wya9irEnvZ6TTz6V4RXF/K5hZkdC0sMRUd9ju2Je8kDS64CvAHngxoi4TtK1wMKIuFPS6cB/AWNq0G4yAAALv0lEQVSAg8CmiJh/qHk6FAaICHaveIj1D9zKmDV3M7E5uR3oypjI0yPOoOXolzPxhFczf/ZMqsrzGRdrZgMiFIrBoTAwNW5Zzrq/3EnbM79h6q6HqSK5l8PTUceq4SfSOOl0Rs09m2OPO54JI4dlXK1Z6XEoWHZamtiz8iE2PXEPudV/ZOKeRVTHfgAaYhTLcrPYPmo+TD6FUTNOZtr0OUwdV00+5+MSZsXiULCBo62Vxg2L2fTk72hc/Rdqti1iQtMa8unPU3bHcJ5hKluqZnBg5ExytbMZUTefo+pmMX1CDTVV5Rk/AbPBr7eh4COCVny5PJV1JzKt7sTnhjXu5cDax9i24lEOrl9E7banmLv/T4xo+BU0AEugMcpYGxN4JDeZXcPqaBwxBUZPo2r8dKonTKN2/ESOGl3F+OpKct7KMOsTDgXLRuUIhs06i7pZZz1/+L5tNG5+iu2rnuTApqfJbX+WuXtWM/rgk1QdOJgExjNJ0/1RycYYy7OMYVd+HPsrx9M0rJa24RNQTS0VoyZROXIC1WNqGTWimjHDKxgzvJyaqnLvqjLrhkPBBpbqcVQe83ImHfPy5w+PgP3badu+ij2bn2Vfwxqadqwjv2s9U/dvZs7B5dQ0PkjFwSbY8cLZ7o7h7IgRrKKGHTGCffmRHCwbSXPZCFrKR9JWWUNUjoTKGnJVo8hX1ZAfPpLyYTWUD6uhsrKKYeV5qsrzDCvPM6wiR2VZ0l9Vnjwuz8u/17BBz6Fgg4ME1ePIVY9j1NTTGNVVmwg4uAv2NRB7N7N/+yYO7tpM456ttO7divZtY/zBHUxs3EFl03IqW/ZQdXAfuYMBPdxWojHKOEAl+6nkQFRygEp2UMGBqKSRCg5QQSMVNKuSllwlrflKWnOVtOYqaMtX0JqrJHJltOUriFza5SuIfDnkKyBfgcoqky5fhvIVqKyCXL6cXFkFypeRy5eRz+fJ50Q+J3JSx+O8RFk+GVaWE7l0fE4kf3MFj9uHFwxrn5/SfnW0BRVMI9JxOaH0ZclJz12+oLBNOjwnoRwdj/O5ZHxezw/R9vk5WLPlULChQ4Jho2HYaDR+NtXTobqnadraoGkvNO6Gxj1wcDc07qblwG6a9u+mef9uWg7uoa1xH21N+8g17qO6eT/VzQdQ835yLQfIte4h33qQXOtBytoak661kVxr35/E0RqihTJayNFCnhbytJKnhRytkaeVHK3puDZyyXBytKV/W8nTGjmaO/rTYc/rz9EW7dOItoLpA6XzE5H+fa5ftKbTBaKto3tu+vb+1oL5tUV7m2QeoRwBRDo+EChHq/JEe1ew/FAyHuUg7W8fH3puvm3tl3rrGJ8HQRs5pFy6HAG5pE063455tc9HQrQvU4hcxzLVMe/2eRWGXbL4iKSDF173p73t84YV9P/tObN43QmT+vQ91ZlDwUpbLgdVI5OuQBkv8p8jAlqbobURWtKutSnpWhrTcU3p+CZoay5o05z0tzYTrU1EayutLU1EazNtrS3Q2kyutZny1mbKopVIh0VbK9HWAq0tRFsrilZoa4W2Fog2iPRxWyuKtnR4c8djpe0UrYhI27UCkQyLtqRt+lcUPk4+CjMTHOGV1YorCUg6ggWgrT2A2oNKdPR31fa54Tk2rf97OOF9Ra3ZoWBWDBKUVSRdZc2RzybtBsUtEiOS8GlLgqTjcXsgRaT9rc/9bZ+moyvop9O4tk4hlwbXc+06/+00j7bW59fZef4R3c/reW07zz+e+9tpHrnC+b1gHp2W94J5dVpGBGPnHlv0l9GhYGZ9QwLlIefLmgxmg+ILiJmZ9Q+HgpmZdXAomJlZB4eCmZl1cCiYmVkHh4KZmXVwKJiZWQeHgpmZdRh0N9mR1ACsPsLJxwNb+7CcocDrpGteLy/kdfJCg2mdTIuI2p4aDbpQeDEkLezNnYdKiddJ17xeXsjr5IWG4jrx7iMzM+vgUDAzsw6lFgo3ZF3AAOR10jWvlxfyOnmhIbdOSuqYgpmZHVqpbSmYmdkhOBTMzKxDyYSCpPMlLZO0XNLVWdeTBUlTJd0naamkxZI+nA4fK+k3kp5J/47Jutb+Jikv6VFJv0j7Z0h6MF0nP5ZUkXWN/UnSaEm3S3oqfb+8tNTfJ5I+mv7fPCnpR5KqhuL7pCRCQVIeuB64AJgHXC5pXrZVZaIF+HhEHAe8BPhguh6uBu6JiNnAPWl/qfkwsLSg/3PAl9N1sgN4TyZVZeerwK8i4ljgJJJ1U7LvE0lTgA8B9RFxPJAHLmMIvk9KIhSAM4DlEbEiIpqAW4ELM66p30XExoh4JH28h+QffQrJuvhu2uy7wEXZVJgNSXXA64Fvp/0CXg3cnjYpqXUiaSTwCuA7ABHRFBE7KfH3Ccnti4dJKgOGAxsZgu+TUgmFKcDagv516bCSJWk6cArwIHBURGyEJDiACdlVlomvAJ8E2tL+ccDOiGhJ+0vt/XIM0ADclO5S+7akakr4fRIR64EvAmtIwmAX8DBD8H1SKqGgLoaV7Lm4kkYAPwE+EhG7s64nS5LeAGyJiIcLB3fRtJTeL2XAqcA3IuIUYB8ltKuoK+nxkwuBGcBkoJpkd3Rng/59UiqhsA6YWtBfB2zIqJZMSSonCYQfRMRP08GbJU1Kx08CtmRVXwZeDiyQtIpkt+KrSbYcRqe7CaD03i/rgHUR8WDafztJSJTy++Q8YGVENEREM/BT4GUMwfdJqYTCQ8Ds9EyBCpIDRHdmXFO/S/eVfwdYGhFfKhh1J/DO9PE7gZ/1d21ZiYhrIqIuIqaTvC/ujYi3AfcBl6TNSm2dbALWSpqbDjoXWEIJv09Idhu9RNLw9P+ofZ0MufdJyfyiWdLrSL4B5oEbI+K6jEvqd5LOAv4ALOK5/eefIjmucBtwNMmb/80RsT2TIjMk6RzgExHxBknHkGw5jAUeBa6IiMYs6+tPkk4mOfBeAawA3kXyJbJk3yeS/gl4C8lZfI8C7yU5hjCk3iclEwpmZtazUtl9ZGZmveBQMDOzDg4FMzPr4FAwM7MODgUzM+vgUDAzsw4OBbNekHRy+luX9v4FfXUJdkkfkTS8L+Zl9mL5dwpmvSDpSpLLJl9VhHmvSue99TCmyUdEa1/XYuYtBRtSJE1PbwrzrfSGKL+WNKybtjMl/UrSw5L+IOnYdPib0xupPC7p9+mlUa4F3iLpMUlvkXSlpK+l7W+W9I30BkYrJL1S0o1pHTcXLO8bkhamdf1TOuxDJBdYu0/SfemwyyUtSmv4XMH0eyVdK+lB4KWS/lXSEklPSPpicdaolZyIcOduyHTAdJLLEJyc9t9GcumBrtreA8xOH59Jct0jSC4DMiV9PDr9eyXwtYJpO/qBm0kudSCSK2nuBk4g+dL1cEEtY9O/eeB3wIlp/ypgfPp4MsklJGpJrlZ6L3BROi6AS9vnBSzjua390Vmve3dDo/OWgg1FKyPisfTxwyRB8Tzp5cNfBvynpMeAbwKT0tF/Am6W9D6SD/De+HlEBEmgbI6IRRHRBiwuWP6lkh4huUbOfJK7AHZ2OvC7SK7G2QL8gOSGNwCtJFe4hSR4DgLflvQmYH8v6zQ7pLKem5gNOoUXJGsFutp9lCO5QcrJnUdExAcknUlyN7bH0ovD9XaZbZ2W3waUSZoBfAI4PSJ2pLuVqrqYT1f3cmh3MNLjCBHRIukMkqt1XgZcRXLZb7MXxVsKVpIiubnQSklvhuSy4pJOSh/PjIgHI+IzwFaSe3HsAWpexCJHktysZpeko3j+DVoK5/0g8EpJ49N7i18O3N95ZumWzqiIuAv4CNCb4DLrkbcUrJS9DfiGpH8AykmOCzwOfEHSbJJv7fekw9YAV6e7mv7lcBcUEY9LepRkd9IKkl1U7W4A/lvSxoh4laRrSK7TL+CuiOjqGv01wM8kVaXtPnq4NZl1xaekmplZB+8+MjOzDt59ZEOepOtJ7sVc6KsRcVMW9ZgNZN59ZGZmHbz7yMzMOjgUzMysg0PBzMw6OBTMzKzD/wMLYOytyVdyEwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-logloss-mean']\n",
    "test_stds = cvresult['test-logloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-logloss-mean']\n",
    "train_stds = cvresult['train-logloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators4_1.png' )\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#可以看到 当n_estimators=98的时候，模型的性能最佳。n_estimators 最小。learning_rate 就固定为0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=StratifiedKFold(n_splits=3, random_state=3, shuffle=True),\n",
       "       error_score='raise-deprecating',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bynode=1, colsample_bytree=0.8, gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=1, missing=None, n_estimators=89, n_jobs=1,\n",
       "       nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=3, silent=None,\n",
       "       subsample=0.3, verbosity=1),\n",
       "       fit_params=None, iid='warn', n_jobs=-1,\n",
       "       param_grid={'max_depth': range(3, 10, 2), 'min_child_weight': range(1, 6, 2)},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#第一轮参数调整得到调整max_depth，min_child_weight\n",
    "\n",
    "xgb2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=98,  #第一轮参数调整得到的n_estimators最优值  98\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "\n",
    "gsearch2 = GridSearchCV(xgb1, param_grid = param_test2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2.fit(X_train , y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'max_depth': 3, 'min_child_weight': 3} -0.06529591395163109\n"
     ]
    }
   ],
   "source": [
    " #最佳解\n",
    "print(gsearch2.best_params_,gsearch2.best_score_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#调整colsample_bytree & colsample_bylevel\n",
    "xgb3 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=98,  #第一轮参数调整得到的n_estimators最优值  98\n",
    "        max_depth=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'colsample_bytree': 0.9, 'subsample': 0.8}, -0.0643888027778803)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3= dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "\n",
    "gsearch3 = GridSearchCV(xgb3, param_grid = param_test3, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3.fit(X_train , y_train)\n",
    "\n",
    "gsearch3.best_params_,     gsearch3.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "D:\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:125: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([22.51122268, 22.48029423, 23.00984693, 23.72709537, 24.21949283,\n",
       "        24.22420812, 22.5594124 , 24.18197354, 24.59788179, 24.77810359,\n",
       "        24.59450936, 24.55774275, 23.15693402, 25.19448678, 26.37310958,\n",
       "        26.34464359, 25.88980238, 25.97164313, 25.76492739, 31.01869837,\n",
       "        27.62510554, 28.78945748, 29.06993635, 28.75894618]),\n",
       " 'std_fit_time': array([0.09481152, 0.77869318, 0.18198651, 0.06033159, 0.09364753,\n",
       "        0.04998367, 0.11686377, 0.11792171, 0.21634454, 0.16265467,\n",
       "        0.0801928 , 0.08706987, 0.14819381, 0.04291477, 0.25566866,\n",
       "        0.06158575, 0.07638223, 0.15697468, 2.58158796, 0.62713718,\n",
       "        0.13654196, 0.29024317, 0.11363895, 0.59081616]),\n",
       " 'mean_score_time': array([0.30634793, 0.32479779, 0.30701987, 0.31859668, 0.31665818,\n",
       "        0.31769204, 0.30595541, 0.33312837, 0.29798985, 0.29279876,\n",
       "        0.29097136, 0.28982186, 0.3237594 , 0.28999718, 0.30578025,\n",
       "        0.31632106, 0.27647638, 0.27888552, 0.30510592, 0.31683723,\n",
       "        0.37336508, 0.32616901, 0.30724589, 0.25611639]),\n",
       " 'std_score_time': array([0.00772765, 0.02412502, 0.01034592, 0.02015173, 0.01454445,\n",
       "        0.03676831, 0.00690106, 0.02112024, 0.01424024, 0.00657759,\n",
       "        0.00836958, 0.01795412, 0.04259996, 0.01422477, 0.01772241,\n",
       "        0.01623596, 0.00804474, 0.00785684, 0.00991152, 0.00873333,\n",
       "        0.03702375, 0.03951684, 0.02141118, 0.03791315]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'split0_test_score': array([-0.06484663, -0.06447011, -0.06440525, -0.06403666, -0.06423285,\n",
       "        -0.0643716 , -0.06514034, -0.06463064, -0.06435554, -0.06417504,\n",
       "        -0.06441821, -0.06435145, -0.06491837, -0.06439571, -0.06445587,\n",
       "        -0.06421747, -0.06426336, -0.06430988, -0.06514297, -0.06455864,\n",
       "        -0.06448097, -0.06422669, -0.06427921, -0.06419756]),\n",
       " 'split1_test_score': array([-0.06585974, -0.0653178 , -0.06518631, -0.0650453 , -0.06506931,\n",
       "        -0.0648903 , -0.06597382, -0.06518125, -0.06499708, -0.0650937 ,\n",
       "        -0.06497718, -0.06483805, -0.06594462, -0.06545798, -0.06483163,\n",
       "        -0.06499522, -0.0650425 , -0.06486134, -0.06588467, -0.06534619,\n",
       "        -0.064855  , -0.06488957, -0.0648845 , -0.06498157]),\n",
       " 'split2_test_score': array([-0.06500107, -0.06483811, -0.06458164, -0.06451509, -0.06440312,\n",
       "        -0.06441437, -0.0653721 , -0.06489335, -0.06477751, -0.06452973,\n",
       "        -0.06467669, -0.06434861, -0.06510919, -0.06485393, -0.06442486,\n",
       "        -0.06451527, -0.0644736 , -0.06413256, -0.06506119, -0.06476696,\n",
       "        -0.0646888 , -0.06448031, -0.06447453, -0.06398727]),\n",
       " 'mean_test_score': array([-0.06523581, -0.06487534, -0.0647244 , -0.06453235, -0.06456843,\n",
       "        -0.06455875, -0.06549542, -0.06490175, -0.06471004, -0.06459949,\n",
       "        -0.06469069, -0.0645127 , -0.06532406, -0.06490253, -0.06457079,\n",
       "        -0.06457598, -0.06459315, -0.06443459, -0.06536295, -0.06489059,\n",
       "        -0.06467492, -0.06453219, -0.06454608, -0.0643888 ]),\n",
       " 'std_test_score': array([0.00044567, 0.00034707, 0.00033447, 0.00041196, 0.00036093,\n",
       "        0.00023509, 0.00035127, 0.00022487, 0.00026622, 0.00037827,\n",
       "        0.00022842, 0.00023006, 0.00044566, 0.00043503, 0.00018488,\n",
       "        0.00032041, 0.00032913, 0.00031031, 0.00037042, 0.00033319,\n",
       "        0.00015301, 0.0002731 , 0.00025224, 0.00042785]),\n",
       " 'rank_test_score': array([21, 17, 16,  5,  8,  7, 24, 19, 15, 12, 14,  3, 22, 20,  9, 10, 11,\n",
       "         2, 23, 18, 13,  4,  6,  1]),\n",
       " 'split0_train_score': array([-0.06235996, -0.06228512, -0.06231594, -0.06191144, -0.06230086,\n",
       "        -0.062214  , -0.06222752, -0.06218004, -0.06213007, -0.06216989,\n",
       "        -0.06215371, -0.06236389, -0.06219738, -0.06204147, -0.06210093,\n",
       "        -0.06211549, -0.06222788, -0.06225009, -0.06216468, -0.06187513,\n",
       "        -0.06195791, -0.06190356, -0.06207476, -0.06216138]),\n",
       " 'split1_train_score': array([-0.06181135, -0.0616925 , -0.06185098, -0.06189448, -0.06195769,\n",
       "        -0.06215517, -0.06165194, -0.06174274, -0.06191062, -0.06182719,\n",
       "        -0.06175305, -0.06193284, -0.06178712, -0.06160798, -0.06160218,\n",
       "        -0.06156961, -0.06198297, -0.06194968, -0.06154192, -0.06144146,\n",
       "        -0.06152182, -0.06159567, -0.061779  , -0.06193931]),\n",
       " 'split2_train_score': array([-0.06228164, -0.06239275, -0.06233037, -0.06227156, -0.06237417,\n",
       "        -0.06249383, -0.06236852, -0.06240378, -0.06225402, -0.06234132,\n",
       "        -0.06238394, -0.06247562, -0.0619115 , -0.06200513, -0.06188   ,\n",
       "        -0.06202683, -0.06207784, -0.06203968, -0.0621683 , -0.06206678,\n",
       "        -0.06217131, -0.06181807, -0.06203693, -0.06203042]),\n",
       " 'mean_train_score': array([-0.06215098, -0.06212346, -0.06216576, -0.06202583, -0.06221091,\n",
       "        -0.06228766, -0.06208266, -0.06210886, -0.06209824, -0.0621128 ,\n",
       "        -0.0620969 , -0.06225745, -0.06196533, -0.06188486, -0.06186104,\n",
       "        -0.06190398, -0.06209623, -0.06207982, -0.0619583 , -0.06179446,\n",
       "        -0.06188368, -0.06177243, -0.06196356, -0.06204371]),\n",
       " 'std_train_score': array([2.42273196e-04, 3.07882389e-04, 2.22666006e-04, 1.73897520e-04,\n",
       "        1.81535044e-04, 1.47743976e-04, 3.09957294e-04, 2.74522795e-04,\n",
       "        1.41988962e-04, 2.13738269e-04, 2.60670643e-04, 2.34023529e-04,\n",
       "        1.71757953e-04, 1.96344809e-04, 2.04054519e-04, 2.39186240e-04,\n",
       "        1.00824762e-04, 1.25884406e-04, 2.94426174e-04, 2.61578428e-04,\n",
       "        2.70296130e-04, 1.29772248e-04, 1.31415494e-04, 9.11467902e-05])}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.064389 using {'colsample_bytree': 0.9, 'subsample': 0.8}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZgAAAEKCAYAAAAvlUMdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xd8jdcfwPHPyRYjSIxIisSqkUVE7b1KtUa1ttZotVqlFFW7Vqka1aFm0WH8qloq1KZIjFihRqwQEkIQiazv7497pREZV+RKcN6v13259z7nnPt9rvDNc57n+R4lImiapmladrPI6QA0TdO0Z5NOMJqmaZpZ6ASjaZqmmYVOMJqmaZpZ6ASjaZqmmYVOMJqmaZpZ6ASjaZqmmYVOMJqmaZpZ6ASjaZqmmYVVTgeQk5ycnKR06dI5HYamadpTZf/+/ddEpEhm7Z7rBFO6dGn27duX02FomqY9VZRS501pp6fINE3TNLPQCUbTNE0zC51gNE3TNLN4rs/BaFpuER8fT2hoKLGxsTkdiqYls7Ozw9XVFWtr6yz11wlG03KB0NBQ8ufPT+nSpVFK5XQ4moaIcP36dUJDQ3Fzc8vSGHqKTNNygdjYWBwdHXVy0XINpRSOjo6PdVStE4ym5RI6uWi5zeP+TOoEkxVXjsLG0aCXm9Y0TUuXTjBZcX4X7JoB//6V05FomqblWjrBZIXv2+BUATaMgIR7OR2Npj22mzdv8s0332Sp74wZM7h79262xbJo0SL69++fbeOZauvWrbRu3fqR+z1qvOfOneOnn3565M8xVWRkJE2bNqVcuXI0bdqUGzdupNnuwoULNGvWjIoVK1KpUiXOnTuX7bHoBJMVltbQYiJEhsDe73M6Gk17bLkpwTzrMkowCQkJjz3+5MmTady4MadOnaJx48ZMnjw5zXbdu3dnyJAhHD9+nICAAIoWLfrYn52avkw5q8o2gXLNYftU8OoE+TKt+6ZpJhn7xzGCL9/K1jErlSjA6Fcqp7t92LBhnDlzBm9vb5o2bUrRokVZvnw59+7do23btowdO5bo6Gg6duxIaGgoiYmJjBw5kqtXr3L58mUaNmyIk5MTW7ZsSXP89evX8+mnn5KYmIiTkxObNm0iMjKSt99+m5CQEOzt7Zk7dy6enp4P9FuxYgVjx47F0tISBwcHtm/fzrlz5+jWrRvR0dEAfP3119SqVYutW7cyevRoihUrRlBQEO3atcPDw4OZM2cSExPD6tWrKVOmDD179sTOzo5jx45x9epVpk+f/tCRS3R0NB988AFHjhwhISGBMWPG8Oqrr6b7/V28eJEWLVpw9uxZOnfuzOjRoxk5ciROTk4MGDAAgBEjRlCsWDF++uknjh8/jre3Nz169KBQoUKsXbuW2NhYoqOj2bx5M1OnTn3o+wdYunQps2bNIi4ujho1avDNN99gaWn5QCy///47W7duBaBHjx40aNCAKVOmPNAmODiYhIQEmjZtCkC+fPnS3bfHoRPM42g+Ab55CTaPhzazcjoaTcuyyZMnc/ToUYKCgtiwYQMrV64kICAAEaFNmzZs376diIgISpQowdq1awGIiorCwcGB6dOns2XLFpycnNIcOyIigj59+rB9+3bc3NyIjIwEYPTo0fj4+LB69Wo2b95M9+7dCQoKeqDvuHHj8Pf3x8XFhZs3bwJQtGhRNm7ciJ2dHadOnaJTp07JRWsPHTrE8ePHKVy4MO7u7vTu3ZuAgABmzpzJ7NmzmTFjBmA4iti2bRtnzpyhYcOGnD59+oHPnTBhAo0aNWLBggXcvHkTPz8/mjRpQt68edPcx4CAAI4ePYq9vT3Vq1enVatW9OrVi3bt2jFgwACSkpL45ZdfCAgIwNPTk2nTpvHnn38Chim23bt3c/jwYQoXLsyGDRs4derUQ99/kSJF+PXXX9m1axfW1ta89957LFu2jO7du9O7d2/effddfH19uXr1Ks7OzgA4OzsTHh7+ULwnT56kYMGCtGvXjrNnz9KkSRMmT578ULJ6XDrBPA6ncuDXF/Z8C9V7g7Nn5n00LRMZHWk8CRs2bGDDhg34+PgAcOfOHU6dOkXdunUZPHgwQ4cOpXXr1tStW9ek8fbs2UO9evWSb9YrXLgwADt37mTVqlUANGrUiOvXrxMVFfVA39q1a9OzZ086duxIu3btAEPVg/79+xMUFISlpSUnT55Mbl+9evXk/1zLlClDs2bNAPDw8Hjg6Kpjx45YWFhQrlw53N3dOXHixEPfwZo1a5g2bRpguE/pwoULVKxYMc19bNq0KY6OjgC0a9eOnTt38tFHH+Ho6MjBgwe5evUqPj4+yW3S6n//e0nv+z98+DD79++nevXqAMTExCRPa82bNy/NcdOTkJDAjh07OHjwICVLluSNN95g0aJF9OrV65HGyYxOMI+r/idw6BdYPxx6/gn6XgbtKSciDB8+nHfeeeehbfv372fdunUMHz6cZs2aMWrUKJPGS+t+CknjMv/U7b777jv27t3L2rVr8fb2JigoiNmzZ1OsWDEOHTpEUlISdnZ2ye1tbW2Tn1tYWCS/trCweOD8RurPSf1aRFi1ahUVKlTIdP8yGq93794sWrSIK1eu8Pbbb6fbP+WRUXrf/+zZs+nRoweTJk3KMJZixYoRFhaGs7MzYWFhaZ5bcXV1xcfHB3d3dwBee+019uzZk+0JRp/kf1x5CkGjEXB+Jxxfk9PRaFqW5M+fn9u3bwPQvHlzFixYwJ07dwC4dOkS4eHhXL58GXt7e7p27crgwYM5cODAQ33TUrNmTbZt28bZs2cBkqfI6tWrx7JlywDDFVxOTk4UKFDggb5nzpyhRo0ajBs3DicnJy5evEhUVBTOzs5YWFiwZMkSEhMTH3l/V6xYQVJSEmfOnCEkJOShRNK8eXNmz56dnAQPHjyY4XgbN24kMjIy+VxP7dq1AWjbti3r168nMDCQ5s2bA5l/X+l9/40bN2blypXJU16RkZGcP//wsixt2rRh8eLFACxevDjNc0fVq1fnxo0bREREALB582YqVaqU4T5mhT6CyQ5Ve0LgfNjwmeHEv7Vdpl00LTdxdHSkdu3aVKlShZYtW9K5c2dq1qwJGE4AL126lNOnTzNkyBAsLCywtrbm22+/BaBv3760bNkSZ2fnNE/yFylShLlz59KuXTuSkpKSz6GMGTOGt956C09PT+zt7ZP/U0xpyJAhnDp1ChGhcePGeHl58d5779G+fXtWrFhBw4YN0z0vkpEKFSpQv359rl69ynfffffAURDAyJEj+eijj/D09EREKF26dPI5k7TUqVOHbt26cfr0aTp37oyvry8ANjY2NGzYkIIFCyaf3/D09MTKygovLy969uxJoUKFHhirWbNmHD9+/KHvv1KlSnz++ec0a9aMpKQkrK2tmTNnDqVKlXrgHMywYcPo2LEj8+fPp2TJkqxYsQKAffv28d133zFv3jwsLS2ZNm0ajRs3RkSoVq0affr0eeTvMTMqrcPU54Wvr69k24qWIVvhx1eh8Sio+3H2jKk9N44fP57u/L6WvXr27Enr1q3p0KGD2T8rKSmJqlWrsmLFCsqVK2f2zzOHtH42lVL7RcQ3s756iiy7uDeACq1g+5dw+0pOR6NpWg4LDg6mbNmyNG7c+KlNLo9LT5Flp2bjYU4N2DQOXsvaTWua9jSrUaMG9+49WN1iyZIleHh45FBED1u0aFGW+vn7+zN06NAH3nNzc+O3335Ls32lSpUICQnJ0mc9K3SCyU6OZeClfvDPLMNlyy5VczoiTXui9u7dm9MhmE3z5s2TT9RrptFTZNmt3hDIW8Rw2fJzfH5L0zTNrAlGKdVCKfWvUuq0UmpYGtttlVK/GrfvVUqVTrHNUym1Wyl1TCl1RCllZ3x/q3HMIOOjaKoxOyilRCmV6Qkos7ArAI1GwsU9cHRVjoSgaZqWG5gtwSilLIE5QEugEtBJKZX6QutewA0RKQt8BUwx9rUClgLvikhloAEQn6JfFxHxNj6S6yAopfIDHwI5e5zu0xWKexjWjImPydFQNE3Tcoo5j2D8gNMiEiIiccAvQOo7fl4F7l/8vhJorAy3wDYDDovIIQARuS4iptxNNR74Asj6Gp/ZwcISWkyBW6Hwz+wcDUXTNC2nmDPBuAAXU7wONb6XZhsRSQCiAEegPCBKKX+l1AGl1Cep+i00To+NNCYklFI+wAsikv7dUIZ2fZVS+5RS++7fxWoWpWtDpVdh51dw67L5PkfTskFuKtev14N5PKasB7Nlyxa8vb2TH3Z2dqxevTrbYzFngkmrKFfqs97ptbEC6gBdjH+2VUo1Nm7vIiIeQF3jo5tSygLDFFumdziKyFwR8RUR3yJFzFxiv+l4SEqEv8eY93M07THlpgTzrMsN68E0bNiQoKAggoKC2Lx5M/b29smFQbOTOS9TDgVeSPHaFUj9q/z9NqHG8y4OQKTx/W0icg1AKbUOqApsEpFLACJyWyn1E4apuN+BKsBW4wFNcWCNUqqNiGTTrfpZUKgU1OoPO76E6n3gheo5For2FPlrGFw5kr1jFveAlmkvPAV6PZjnbT2YlFauXEnLli2xt7dPt02WiYhZHhiSVwjgBtgAh4DKqdq8D3xnfP4msNz4vBBwALA3jvM30Mr43MnYxhrDeZt30/jsrYBvZjFWq1ZNzC72tsjU8iJzG4kkJpr/87SnUnBw8H8v1g0VWfBy9j7WDc3w88+ePSuVK1cWERF/f3/p06ePJCUlSWJiorRq1Uq2bdsmK1eulN69eyf3uXnzpoiIlCpVSiIiItIdOzw8XFxdXSUkJERERK5fvy4iIv3795cxY8aIiMimTZvEy8tLREQWLlwo77//voiIVKlSRUJDQ0VE5MaNGyIiEh0dLTExMSIicvLkSbn/73jLli3i4OAgly9fltjYWClRooSMGjVKRERmzJghAwYMEBGRHj16SPPmzSUxMVFOnjwpLi4uEhMTI1u2bJFWrVqJiMjw4cNlyZIlyZ9brlw5uXPnTpr7t3DhQilevLhcu3ZN7t69K5UrV5bAwEA5e/as+Pj4iIhIYmKiuLu7y7Vr1x74nPv9XVxckr+X9L7/4OBgad26tcTFxYmISL9+/WTx4sUiItKrVy8JDAwUEREHB4cH4itYsGC6fzciIg0bNpQ//vgj3e0P/GwaAfvEhDxgtiMYEUlQSvUH/AFLYIGIHFNKjTMGtwaYDyxRSp3GcOTyprHvDaXUdCAQw5TZOhFZq5TKC/grpayNY/4N/GCufcgWtvmgyWhY3Q+OrACvN3I6Ii23y+BI40nQ68E8++vB3BcWFsaRI0fMdgOpWe/kF5F1wLpU741K8TwWeD2dvksxXKqc8r1ooJoJn9sgC+E+EomLQ9nYmNbY800ImGs4F1OxNdg8evVXTXtSRK8H88yvB3Pf8uXLadu2LdbW1hmOmVX6Tv4suLlqFSGvvkZiBms6PMDCAlpMhtuXYecM8wanaVmg14N5vtaDue/nn3+mU6dOGe7b49C1yLLAxt2duAsXuDJ2HCWmfpHmb2cPKfkSVGlvqFNWtRsULGn+QDXNRHo9mOdrPRgwXM128eJF6tev/8jfn6n0ejBZXA/m2rffEjFzFs6TJ1HwtddM63TzInxdHSq0hNcXZulztWeTXg/mydHrwTwavR5MDnDs2xd7X1+ujhtPXBqHqWkq+ALU/hCO/Q/O7zZvgJqm5Si9How+gnmsFS3jw8IIea0tNiVLUnrZUtNO+sdFw2xfyFcU+mwxnJ/RnnvPyhHM07AeTFY96nowz4rHOYLR52Aeg7WzM87jxnFpwAAiZs+m6McmLJVskxeajoX/9YFDPxkKY2raM0KvB6OlpH99zqK78YbSGAWaN6Pg669zfd58onebOO3l8Tq4VjesfHnPxCvRNE3TnjI6wWTB8n+X89rvr3Et5hoAxYYPw8bNjcufDCUhjcJyD1HKUG35zlVDGRlN07RnkE4wWeDh5MGN2Bt8vPVj4hPjsbC3x+XLaSTevEnYpyPSvIHsIa7VDDdg7p4DkWfNH7SmadoTphNMFlR0rMjYWmM5EH6AyQGGsh52FStSdPDH3NmyhRs//2zaQE1Gg4UVbBxpxmg1LXO5qZqyLtf/eEwp1w/wySefULlyZSpWrMiHH35o2i/Gj0gnmCx62f1l3qryFstPLmfFScONTIW6dydvvbqET/mC2BT1kdJVoATUGQTH/4CzO8wcsaalLzclmGddbijX/88//7Br1y4OHz7M0aNHCQwMZNu2bY/92anpBPMYBvgMoLZLbSbuncjB8IMopSgxaRIW+fNz+eOPSYo1YWHNWv3BoSSsH25YO0bTckDKcv1Dhgxh6tSpVK9eHU9PT0aPHg0YSti3atUKLy8vqlSpwq+//sqsWbOSy/U3bNgw3fHXr19P1apV8fLyonFjw9JOkZGRvPbaa3h6evLSSy9x+PDhh/qtWLGCKlWq4OXlRb169QDDf9B169alatWqVK1alX/++QcwHIHUr1+fjh07Ur58eYYNG8ayZcvw8/PDw8ODM2fOAIYbLd99913q1q1L+fLl07xDPzo6mrfffpvq1avj4+PD77//nuH3d79cf4UKFZJL648cOZKZM2cmtxkxYgSzZs1i2LBh7NixA29vb7766isWLVrE66+/ziuvvJJcnDOt7x8M5fr9/Pzw9vbmnXfeSbNMzu+//06PHj0AQ7n+tBYSU0oRGxtLXFwc9+7dIz4+nmLFimW4j1mhL1N+DJYWlkypO4XOazszcMtAfmn9C8Udi1Ni0iQu9ulD+BdTKT4qk+kv6zyGy5ZXvgUHl0C1nk8kdi33mhIwhRORJzJv+AheLPwiQ/2Gprt98uTJHD16lKCgIDZs2MDKlSsJCAhARGjTpg3bt28nIiKCEiVKsHbtWgCioqJwcHBg+vTpbNmyBScnpzTHjoiIoE+fPmzfvh03N7fkWmSjR4/Gx8eH1atXs3nzZrp3705QUNADfceNG4e/vz8uLi7cvHkTILnUjJ2dHadOnaJTp07cv5/t0KFDHD9+nMKFC+Pu7k7v3r0JCAhg5syZzJ49mxkzDLUAz507x7Zt2zhz5gwNGzbk9OnTD3zuhAkTaNSoEQsWLODmzZv4+fnRpEmTdMvSBAQEcPToUezt7alevTqtWrWiV69etGvXjgEDBpCUlMQvv/xCQEAAnp6eTJs2LTmxLVq0iN27d3P48GEKFy7Mhg0bOHXq1EPff5EiRfj111/ZtWsX1tbWvPfeeyxbtozu3bs/UCrm6tWryRWlnZ2dk2uXpVSzZk0aNmyIs7MzIkL//v3Nch+WPoJ5TA62DsxqNIuYhBg+2vIRsQmx5Ktbh8I9e3Ljp5+4vXlz5oNUbgsla8Gm8RAblXl7TTOjlOXiq1atyokTJzh16hQeHh78/fffDB06lB07duDg4GDSeBmV6+/WrRuQebn+H374Ifm39fj4ePr06YOHhwevv/46wcHBye3vl+u3tbV9qFz/uXPnktuZUq5/8uTJeHt706BBg+Ry/em5X64/T548yeX6S5cunVyu//73+ajl+lN+/5s2bUou1+/t7c2mTZsICQkBDOX679c/M8Xp06c5fvw4oaGhXLp0ic2bN7N9+3aT+5tKH8FkgzIFyzCp7iQGbBnAuN3jmFBnAkUGDSQ6YC9hn47A7vfVWGd0+KkUtJgEcxvA9qnQ7PMnFruW+2R0pPEk6HL9z365/t9++42XXnqJfPnyAdCyZcvkXwSykz6CySaNSjbiPe/3+CPkD5YEL8HCxgaXaV+SdO8el4cOQ5KSMh6ghDf4dIE938H1M08maE0z0uX6n69y/SVLlmTbtm0kJCQQHx/Ptm3b9BRZbveO5zs0LtmYL/d/ye7Lu7F1d6P4iE+5u2cP1+fPz3yARqPAyhY2fGb+YDUthZTl+jdu3Jhcrt/Dw4MOHTpw+/Ztjhw5knyCecKECXz2meHn9H65/vRO8qcs1+/l5cUbbxhWdR0zZgz79u3D09OTYcOGpVuu38PDgypVqlCvXr3kcv2LFy/mpZde4uTJk49Vrr9ly5bpluuPj4/H09OTKlWqMHJkxudS75fr9/b2pn379g+V6+/YsWOa5fq/+uqrh8Zq1qxZmt9/ynL9np6eNG3alLCwMMBwpHT/PNSwYcPYuHEj5cqVY+PGjQwbNgwwlOvv3bs3AB06dKBMmTJ4eHjg5eWFl5cXr7zyyiN/j5nRxS4fo9hlWqLjo+m6risRMRH83OpnXPO5cmngIG7//Telf1pGHk/PjAfYMR02jYVuq6FM+lflaM+WZ6XY5dNAl+t/NLpcfy6S1zovMxvOJEmS+HDzh8QkxOA8dgxWRYtwafAQEu9EZzzAS+9BwVKGy5YTH/+aeE3TcoYu16+PYLL9COa+fy79Q79N/QxTZvW/JObAAc53647DK69QYsrDNz49IHgNLO8GL08Dvz5miU/LXZ6VIxhdrv/Zo8v150K1XGoxsOpAvtz/JXMPz+Wdau/g1K8f1+bMIW+dOji8kkFJioqvQOm6sGUieHSAPIXSb6tpuYgu16+lpKfIzKhH5R60cm/F10Ffs/XiVpz6vUueqlW5MmYMcRcvpt/x/mXLsTdh65QnF7CmaVo20gkmCxKThKOXMr8hUinFmJpjqFi4IsN2DOPsnQu4TP0CLCy4NHgwEh+ffufiHlC1OwT+ABEm1DXTNE3LZXSCyYKZf5+k3bf/8M+Za5m2tbOyY2bDmdha2vLhlg+JKZIf53FjiT10mIg5czLu3PAzsLYH/0+zKXJN07QnRyeYLHirthulHe3ps3gfBy9kvsCYcz5npjeYzqXblxi6fSh5mzfDoX07rn8/l+i9Ael3zFcE6n8CpzfCqY3ZuAeapmnmpxNMFhTKa8PSXjVwym9Lz4WBHA+7lWmfasWqMcxvGDsv7WT2wdkU//RTbEqV4vInn5BoLOKXJr93oHAZw1FMYgZTapr2GHJTuX69HszjMXU9mKFDh1KlSpXkytjmYNYEo5RqoZT6Vyl1Wik1LI3ttkqpX43b9yqlSqfY5qmU2q2UOqaUOqKUsjO+v9U4ZpDxUdT4/iClVLBS6rBSapNSqpQ5961oATuW9qpBHmtLus0PICTiTqZ9OlboSPty7Zl/dD4bwndQ4stpJERGEjZyZPqL/VjZQPMJcO0kBM7L5r3QNIPclGCedblhPZi1a9dy4MABgoKC2Lt3L1OnTuXWrcx/UX5UZkswSilLYA7QEqgEdFJKVUrVrBdwQ0TKAl8BU4x9rYClwLsiUhloAKT89b2LiHgbH/drUR8EfEXEE1gJfGGePfvPC4XtWdq7BiJC13l7uXQzJsP2SilG1BiBdxFvRu4ayXlnS4oOHMjtjX9z89fl6Xcs3wLcG8LWSRB9PZv3QtP0ejCpPevrwQQHB1O/fn2srKzImzcvXl5erF+/PsN9zBIRMcsDqAn4p3g9HBieqo0/UNP43Aq4BijgZWBpOuNuxZBIMvpsH2BXZjFWq1ZNssPRSzelyuj1Uv+LzXL1Vkym7SPuRkij5Y2k2Ypmci06Qs6/3UuOe3lL7KlT6Xe6GiwyppDIn4OyJWYtdwkODk5+HjZhgpzr2i1bH2ETJmT4+WfPnpXKlSuLiIi/v7/06dNHkpKSJDExUVq1aiXbtm2TlStXSu/evZP73Lx5U0RESpUqJREREemOHR4eLq6urhISEiIiItevXxcRkf79+8uYMWNERGTTpk3i5eUlIiILFy6U999/X0REqlSpIqGhoSIicuPGDRERiY6OlpgYw7+zkydPyv1/x1u2bBEHBwe5fPmyxMbGSokSJWTUqFEiIjJjxgwZMGCAiIj06NFDmjdvLomJiXLy5ElxcXGRmJgY2bJli7Rq1UpERIYPHy5LlixJ/txy5crJnTt30ty/hQsXSvHixeXatWty9+5dqVy5sgQGBsrZs2fFx8dHREQSExPF3d1drl279sDn3O/v4uKS/L2k9/0HBwdL69atJS4uTkRE+vXrJ4sXLxYRkV69eklgYKCIiDg4ODwQX8GCBR+K2d/fX2rVqiXR0dESEREhbm5uMm3atDT3L+XP5n3APjEhD5hziswFSHmzR6jxvTTbiEgCEAU4AuUBUUr5K6UOKKU+SdVvoXF6bKRKqw644cjor+zYCVNULuHAorf8CL99j+7zA7h5Ny7D9k55nJjZcCbXYq7xyY6hFJk4Hgt7ey59PJikVHdBJytaEXzfhn0L4Gpw2m00LRvo9WCe/fVgmjVrxssvv0ytWrXo1KkTNWvWxMoq+++7N+ed/Gn9x5/6REN6bayAOkB14C6wyViaYBOG6bFLSqn8wCqgG/Bj8oBKdQV8gfppBqVUX6AvGEpWZ5dqpQrxQ3df3loYSI+FgSzrXYN8tul/vVWcqjC61mhG7BzBzPOL6T9pIhffeZfwaV9SfEQ6lyU3/BSOrAD/4YZimGnmVu1pV/zTnL0sXfR6MM/8ejBgmLIbMWIEAJ07dzZLvTRzHsGEAi+keO0KXE6vjfG8iwMQaXx/m4hcE5G7wDqgKoCIXDL+eRv4CfC7P5hSqgkwAmgjImkeCojIXBHxFRHfIkWKPPZOplS7rBNzulTl6KUoei8OJDY+43Uq2pRpQ9eKXVl2fBkbS0RSqHs3bixZwu2tW9PuYF8YGgyHkK3w7xM7QNOeA3o9mOdrPZjExESuXzeczz18+DCHDx9OPtrLTuZMMIFAOaWUm1LKBngTWJOqzRqgh/F5B2CzcX7PH/BUStkbE099IFgpZaWUcgJQSlkDrYGjxtc+wPcYksvDi1A/IU0rFWN6Ry/2no3kvWUHiEvIeKGxj30/poZzDcbvGU9492bYvvgiYcM/JT6NdbQBqN4LnCrAhhGQkPFUnKaZSq8H83ytBxMfH0/dunWpVKkSffv2ZenSpWaZIjPbSX5j5n8ZOAmcAUYY3xuHIQkA2AErgNNAAOCeom9X4BiGBPKF8b28wH7gsHHbTMDSuO1v4CoQZHysySy+7DrJn5Zle85LqaF/ynvL9ktCYlKGbW/E3JDmK5tLo18byeUjAXLcy1vOv/W2JCUmpt3h5EaR0QVEds0yQ+RaTkjrRKpmHj169JAVK1Y8kc/H43yQAAAgAElEQVRKTEwULy8vOXny5BP5PHN4nJP8Zq2mLCLrMExvpXxvVIrnscDr6fRdiuFS5ZTvRQPV0mnf5HHjzU6da5Qk+l4CE9YdJ6+NJZPbeWJhkfY5k4J2BZnZcCbd/urG4AszmDl0CBFjxxO5cBGOvdKYty3XBMo1g21fgOebhjv+NU3LVYKDg2ndujVt27Z9bteD0eX6zahPPXdu30tg1qZT5LW1YlTrSmme7ASoULgCn9f+nI+3fczssmXo3bQp4TNmYF+jBnmqVH64Q7MJ8G1N2PI5vDLz4e2algOehvVgFi1alKV+j7oeTKVKlZKv8npe6QRjZgOblONObAILdp0lv501g5qWT7dts9LN6BPZhx+O/EDl7gPxOXKEyx9/jNv/VmGRep65SHmo3gcCvofqvQ3VlzUth+n1YLSUdC0yM1NKMbJ1Rd7wfYFZm04xd/uZDNv39+lPfdf6TDo+m1vD3ibuwgWuTJiYduMGQ8GuoGF55ed4ZdJnhei/Qy2XedyfSZ1gngClFBPbedDK05mJ607w0970b9iyUBZMqjsJ1/yuDLz5A7ZvdSbqf//j1rp1DzfOU8hwb8y5HXD8DzPugWZudnZ2XL9+XScZLdcQEa5fv/7QFXaPQj3PP9C+vr5y/9K+JyEuIYl3l+5ny7/hzHjDm1e9Uxc2+M/ZqLN0XtuZUvauTPxZER9yDrfffsPGNVWfxAT4vi7ERcP7AWCd9R8GLefEx8cTGhpKbGxsToeiacns7OxwdXXF2tr6gfeNN75nWjpAJ5gnmGAAYuMT6bkwgMBzN/i2S1WaVS6ebtvtodvpv6k/HfLV441Je7EtV45SS35Epb5e/cwWWPIaNB4NdQeZeQ80TXvemZpg9BTZE2Znbcm8HtWp4uJA/58OsvNU+qti1nOtx4dVP2TFnW2c6N2AmIMHufbtdw83LNMQKrwMO76E21fMGL2maZrpdILJAflsrVj8VnXci+Slz4/72H8+Mt22var0onnp5ozM409s05pc+/Zb7qZ11NXsc0i4B5vGmzFyTdM00+kEk0MK2tvwYy8/ijvY0XNhIMcuR6XZTinFuFrjKFeoHB9XPY5yKc6lIZ+QmKrqLI5l4KV3IWgZXM64bpKmadqToBNMDiqa346lvWuQ39aK7vMDOB2e9qqY9tb2zGw4k/g8VsxuY0VCRDhho0Y/fMVRvSFg7wh/DdOXLWualuN0gslhLgXzsKzPSyil6DpvLxcj01561jW/K9PqT2OXw1V2t3bjtr8/UatWPdjIzgEaj4SLe+DY/55A9JqmaenTCSYXcHPKy5JefsTEJ9Jl3l6u3kr7UtUazjUY7DuYGS+eJbKKK1cmTORe6lIUPt2gmAdsHA3xGS/hrGmaZk46weQSFZ0LsOit6ly/c4+u8/YSGZ12Kf4uFbvwStlXGd4gjARrC8MqmHEp2lpYQsvJEHUR/pn9hKLXNE17mE4wuYhPyULM61GdC5F36bEggNux8Q+1UUoxquYoXN08mdUyiXvHjxMxPdWaEqXrQMU2sPMruJV6jTdN07QnQyeYXKZmGUe+7VqV42G36LVoHzFxD6/WZ2tpy1cNvuJUZQd21shP5KJF3Nmx48FGzcZDUiL8PebJBK5pmpaKTjC5UKMXizHjTW/2nY/knaX7uZfwcJIplrcYXzX4ih/qx3HN2Z7Lw4aRcC3FTZuFSkPN9+Hwr3Ax8MkFr2maZqQTTC7V2rMEk9t5sv1kBB/9EkRC4sNLL3sX9WZonZFManWPuFtRXB7+KZKUol3dQZCvGKwfBkkZL92saZqW3XSCycU6Vn+Bka0r8dfRKwxddYSkpIfvbWlfvj216rzJwoZC9I4d3Fiy5L+NtvkN9cku7YOjK59g5JqmaTrB5Hq96rgxqGl5Vh0IZewfx9Is5z7UbyjXW/qyv7wlV6ZNIzY4+L+NXp3A2dtw2XJc9BOMXNO0551OME+BDxqVpW89dxbvPs+0Df8+tN3awpovG0xnZfuiRNklceHjQSTdNd6waWEBLSbD7cuwSy+trGnak6MTzFNAKcXwli/Sya8kc7ac4Zutpx9q45jHkYmtv+bbNjYknDtP2MQUq2CWqgmV2xkSzM2LTzByTdOeZ5kmGKVUGaWUrfF5A6XUh0qpguYPTUtJKcXnr1XhVe8SfLH+X5bsPvdQm0qOlXizywRWv6S4tXIVt9b7/7ex6TjDn3+PfiLxapqmmXIEswpIVEqVBeYDbsBPZo1KS5OlhWLa6140qViMkb8fY9X+0IfavOz+MnneeZvTznBhxDDiw8IMGwq+ALU+hKOr4MKeJxy5pmnPI1MSTJKIJABtgRkiMhBwNm9YWnqsLS34urMPtcs6MmTlIdYfDXuozYd+A9nZx4+4+FhOfvQekmi8j6bOR5C/BPw1VF+2rGma2ZmSYOKVUp2AHsCfxvesM2ivmZmdtSVzu/ni/UJBPvj5INtORjyw3dLCkk/bz+K3V4tgcegE52Z/adhgkxeajIGwIDj08xOPW9O054spCeYtoCYwQUTOKqXcgKXmDUvLTF5bKxa+5Ue5ovl5Z8k+As89uCqmg60DPQYt4J8q1tydu5CofXsNGzxeBxdf2DQW7t3Ogcg1TXteZJpgRCRYRD4UkZ+VUoWA/CIy+QnEpmXCIY81P/byo0TBPLy9MJAjoQ+uclm2UFncx08iogCc/Og9Em7dMly23HIK3LkKO6bnUOSapj0PTLmKbKtSqoBSqjBwCFiolDLpfyalVAul1L9KqdNKqWFpbLdVSv1q3L5XKVU6xTZPpdRupdQxpdQRpZRdinj+VUoFGR9FMxvrWeaUz5ZlvWtQII813Rfs5dTVB49KGlZsxaWPXyfP9bvsGfSW4UZNV1/wfAN2z4Eb53ImcE3TnnmmTJE5iMgtoB2wUESqAU0y66SUsgTmAC2BSkAnpVSlVM16ATdEpCzwFTDF2NcKwzTcuyJSGWgApKxd30VEvI2P8IzGeh44O+Thpz41sLa0oMu8vVy4/uCqmJ06jGHfK2Vx3BnMgUXG8zFNxhjWjtkw8onHq2na88GUBGOllHIGOvLfSX5T+AGnRSREROKAX4BXU7V5FVhsfL4SaKyUUkAz4LCIHAIQkesi8nBJYdPGei6UcszL0t41iEtMovO8PVyJ+m9VTAtlQYdxywhxt8di+nzOH9sDBUpAnYFwfA2c25mDkWua9qwyJcGMA/yBMyISqJRyB06Z0M8FSHnbeKjxvTTbGC+FjgIcgfKAKKX8lVIHlFKfpOq30Dg9NjJFEklvrOdG+WL5+fFtP27ejafLvD1cv3MveVs+uwJUnjmXBCvFqQ/7EX03Cmp9AA4vwF/DDGvHaJqmZSNTTvKvEBFPEelnfB0iIu1NGDuto4fUlRrTa2MF1AG6GP9sq5RqbNzeRUQ8gLrGR7dH+DyUUn2VUvuUUvsiIiLS6PJ083QtyIKe1bl0M4buCwKIivlvZrF0uWow7D1cLsXyx9BOiJUdNB0LV4/AwSUZjKppmvboTDnJ76qU+k0pFa6UuqqUWqWUcjVh7FDghRSvXYHU6/cmtzGed3EAIo3vbxORayJyF1gHVAUQkUvGP29jqCjgl8lYDxCRuSLiKyK+RYoUMWE3nj5+boX5rms1Tl69zduLArkbl5C8rfob/bnazBuvjWdZsewzQ42ykjVh03iIjcpgVE3TtEdjyhTZQmANUALDNNQfxvcyEwiUU0q5KaVsgDeN46S0BsMNnAAdgM1iqEfvD3gqpeyNyaI+EKyUslJKOQEopayB1sDRTMZ6LjWoUJRZb/pw8MIN+v64n9j4/6bA6k6ezw3n/Lh+9T+2H10LLSbB3euwfWoORqxp2rPGlARTREQWikiC8bEIyPRXf+N5kP4YksVxYLmIHFNKjVNKtTE2mw84KqVOA4OAYca+N4DpGJJUEHBARNYCtoC/Uuqw8f1LwA8ZjfU8a+nhzBcdvNh5+hof/HyQeOOqmJb29lT++gfy3VNc+nQ4Z+wLgHcX2PMdXD+Tw1FrmvasUJn9kq+U+htYBNyvLdIJeEtEGqfb6Snh6+sr+/bty+kwzG7xP+cYveYYr3mXYHpHbywsDKerzs2bQ8y0r1n9ihPvD1tEge/rgVs96KTLyGialj6l1H4R8c2snSlHMG9juET5ChCGYfrprccLT3uSetQqzZDmFVgddJmRvx9NXhWzVK/3SHjJm5fXXWPqn2NJrDMQ/l0HZ7bkcMSapj0LTLmK7IKItBGRIiJSVERew3DTpfYUeb9hWfo1KMOyvReY/NcJRASlFBWnz0EVyEfdHwL5xiIRCpYC/08hMSHzQTVN0zKQ1RUtB2VrFNoT8UnzCnR7qRTfbw9hzhbDqphWhQvjPnUGL1yDxDmLWe/bEcKD4cCinA1W07SnXlYTzHNzh/yzRCnF2DaVaefjwrQNJ1m46ywA+erUxuGtHjQ/KKz2X8W/pfxg8wSIuZHDEWua9jTLaoJ5bi//fdpZWCi+6OBJ88rFGPtHMMv3GYotOA8chFXFCvT+8x6j4uO4cS8Ktn2Rw9FqmvY0SzfBKKVuK6VupfG4jeGeGO0pZWVpwaxOPtQt58SwVYdZezgMZWNDyelfYY8Nr6+6xidulUgImAsRJ3M6XE3TnlLpJhgRyS8iBdJ45BcRqycZpJb9bK0s+b5bNaqVKsRHvx5ky4lwbN3cKPHZKCqfT6L4zht86egIG0bkdKiapj2lsjpFpj0D7G2smN+zOhWK5+fdpfvZE3Idh3ZtKfByS97cIeyJsuH3sF1w6u+cDlXTtKeQTjDPuQJ21vz4dg1KFran16JADoVGUXzMGGyKO/PJWium5i/MkY1DITE+88E0TdNS0AlGo3BeG5b2roFjPlt6LAjg1F1wmTaNgjfieWeTFR/Z3iVi96ycDlPTtKeMTjAaAMUK2LGsdw3srC3oOi+AqyXL4/T+e/gevofHCUsGHp9H3O0rOR2mpmlPEVPK9ad1NdlFYwl/9ycRpPZkvFDYnmW9a5AkQtd5e4l7swd5fKvRZ6MFV6ItGf9nV0Tf4a9pmolMOYKZDgzBUKrfFRiMoYLxL8AC84Wm5YSyRQ2rYt6Kjafbwn3Yjf4cK5s8fP6nNX/EXGHcwhokndmc02FqmvYUMCXBtBCR70XktojcEpG5wMsi8itQyMzxaTmgiosDC3tWJywqlp5rL+Dw2SgcQmOZHODMKqt7jPqrF4nL3oBrpqycrWna88qUBJOklOqolLIwPjqm2Kbv6H9G+ZYuzNzu1QiJiObdiwXJ+3pHSm0N5fvVRQmMy8untw6S8M1LsO4TuPvQwqGapmkmJZguGNa9Dzc+ugFdlVJ5MCwopj2j6pYrwuzOPhy5FMVQ1xY4jRtH4bC7TF8A9vutGe7mRXzgDzDLG/75GhLu5XTImqblIpkuOPYse14WHHtcvx0MZdDyQ9RwK8wQv6IUX/Itt9auJdQR9nWqxMcOSdic2QyF3KDpOKj4CihdD1XTnlXZtuCYUsrVeMVYuFLqqlJqlVLKNXvC1J4GbX1cmdLek+DLt2j/ywned29L2PCJFLFw4LWvg/l96y3uvrIQrOxgeTdY+DJcOpDTYWualsNMWTJ5I/ATsMT4Vlegi4g0NXNsZqePYB5N9L0EVh0IZcHOs5y7fhe3vBb0DV2C5/aD3M1vjdvIsTgWj0RtmQh3r4Hnm9B4FDi45HTomqZlI1OPYExJMEEi4p3Ze08jnWCyJilJ2HwinHk7Q9gTEkkV+Zt+e9fjfgXy1K+Ly/AhWJ9eBru/AWUBtT6A2gPANl9Oh65pWjbItiky4JpSqqtSytL46Apcf/wQtaeVhYWiSaVi/NK3Jn9+UIfy1XowuFlnFjey5OY/OznZ9g2uX3JH+u2BF1+G7V/A7KpwYAkkJeZ0+JqmPSGmHMGUBL4GamK4LPkf4EMRuWD+8MxLH8Fkn6u3Yhm/5ReCz06n119WVD0fQ2yZCpT7YiL5CtwB/08hNBCKeUDzCeBeP6dD1jQti7JtiiydwT8SkRlZiiwX0Qkm+60P+Zuh2wdT+2h+um6MoUBcDJeatcNv5GAKXfkbNo6BqAtQviU0Gw9O5XI6ZE3THlF2TpGlZVAW+2nPuBbuTfi6yWz2et1l6sBiHKriR0n/lQQ1b8U3f8VyrtNWaDIGzu0EfaOmpj3Tsppg9E0OWrrqutbl68ZfE2oVzvIukdyeMok8djY0nP85f3YbwAdHPAlsswnx6Q4P3KgZl9Oha5qWjbI6RXZBREqaIZ4nSk+RmVfglUDe3/Q+Re2L8kP9b7g3fyXRixYSbWXL95VfIcyvIQO9k2h8cRYWZzbpGzU17Snx2OdglFK3SbvWmALyiIjV44WY83SCMb+D4Qfp93c/CtkWYn7z+TiGRXNp5CjuBQXxr8uLTKn4GonOLnxW4RKtrnyD5bUTUKq24UKAEj45Hb6maWl47HMwIpJfRAqk8chvanJRSrVQSv2rlDqtlBqWxnZbpdSvxu17lVKlU2zzVErtVkodU0odUUrZpeq7Ril1NMVrb6XUHqVUkFJqn1LKz5QYNfPyKerDD01/ICouip7rexJezA63n5ZRfMxoKkaFMn/7dHqe387AgEJ4Xh3JatfBJISfgLkN4H/vQNSlnN4FTdOyyGwrWiqlLIE5QEugEtBJKVUpVbNewA0RKQt8BUwx9rUClgLvikhloAGQvCi8UqodcCfVWF8AY403gI4yvtZyAY8iHsxrNo+7CXfp6d+Tc7fPU+jNN3Ffu5YCDRrQcPsK/jz6A2853uOTc75435jC2gJvknT0f8jsarB5AtxL/detaVpuZ84lk/2A0yISIiJxGBYoezVVm1eBxcbnK4HGSikFNAMOi8ghABG5LiKJAEqpfBiuYvs81VgCFDA+dwAuZ/P+aI+hkmMlFjRfQEJSAm+tf4szN89gXaworrNm4jrna6yib9Pq+5FssAzk3TrlGX23A/ViprJNVYftXyD6Rk1Ne+qYM8G4ABdTvA41vpdmGxFJAKIAR6A8IEopf6XUAaXUJyn6jAe+BO6mGusjYKpS6iIwDRieVlBKqb7GKbR9ERERWdszLUvKFyrPguYLUErxtv/b/Bv5LwD5GzfGfe2fFOrShZjlv9B82kdsqKEY0L4xk/MOod29MRy54wBr+pPwbV0I2ZbDe6JpminMmWDSugwo9UUD6bWxAupgWIumDtBWKdVYKeUNlBWR39Lo1w8YKCIvAAOB+WkFJSJzRcRXRHyLFCli4q5o2aVMwTIsbL4QKwsrem3oRfD1YAAs8+Wj+GcjKP3Lz1gWKMDVDz6g5o/TWNOpAh/36sZXJb+mf9wHXAm/Cj+24c7CDnpFTU3L5cyZYEKBF1K8duXhaavkNsbzLg5ApPH9bSJyTUTuAuuAqhjK1VRTSp0DdgLllVJbjWP1AP5nfL4CwxSdlguVdijNohaLyGuVl97+vTkccTh5Wx4vL9xWraTIoEHc2baNs61aU2nf3yzoWZ2PPhrGD56/Mi2pE3JuJwlf1yD0pw+QaF0aT9NyI3MmmECgnFLKTSllA7wJrEnVZg2GxADQAdgshuum/QFPpZS9MfHUB4JF5FsRKSEipTEc2ZwUkQbG/peN7QAaAfrX21zshfwvsLDFQhxsHei7sS8Hrv63foyytsapbx/c1/yOnUcVrowZy/kuXXkhKoyx7X3pNXQWK2qt4XeLxjj/u4Q7Uz04+Mt4YmNjcnCPNE1LzawrWiqlXgZmAJbAAhGZoJQaB+wTkTXGS4+XAD4YjlzeFJEQY9+uGM6jCLBORD5JNXZp4E8RqWJ8XQeYiWF6LRZ4T0T2ZxSfvg8m512NvkrvDb25evcqcxrPoXrx6g9sFxGifv+d8MlTSIyOxqlPbxzfeQcLW1viEpLYtnM7hXaNwzd+PxcpzsEKA3mpVQ+KFsiTQ3ukac8+sxa7fFboBJM7XIu5Rm//3ly6c4mZjWZSq0Sth9okREYSPmUKUb+vwaZ0aYqPG0teP8MsqIhwYsdvFNgxFpf4cwQkVWRX2UG0aNqCis4FHhpL07THoxOMCXSCyT0iYyPps6EP56LO8VXDr6jnWi/Ndnd27eLKmLHEX7yIQ4f2FBs8GMuCBQ0bExO4tmMedjsnkS/hJqsS67LV5R3aNvCjQfmiWFjo8jOalh10gjGBTjC5y83Ym/Td2JdTN08xrf40GpdsnGa7pJgYrn3zDdcXLMSyYEGKfTqcAi+/jLpfvyz2FrFbpmEV8C0JAt8ntMa/YEc616lE+6qu5LGxfIJ7pWnPHp1gTKATTO5zK+4W/Tb2I/h6MJPrTaZ56ebpto09fpywUaOJPXKEvPXqUnzUaGxcU9xqdeM8SX+PweLY/7huUZjJ9zqwyaYRnV5yo3vN0hQrYJfu2JqmpU8nGBPoBJM73Ym7w/ub3icoIoiJdSbSyr1Vum0lMZEby5YRPmMmiFDkww8p3K0ryipFubyLAYj/p6jQQC7YlGXYnTcIVFV4xbMEb9dxo4qLwxPYK017dugEYwKdYHKvu/F3+WDzBwReCWRc7XG8Vva1DNvHX77MlXHjubN1K3aVKlF8/DjyVK78XwMROPa/5BU1TzjU4eMb7TkWV4waboXpXdedxi/q8zSaZgqdYEygE0zuFpMQw4DNA9gdtptRNUfxevnXM2wvItz29+fK5xNIjIykcI8eFPmgPxb29v81io+Fvd/C9i+RhBiOlXidIREtOB5lTWlHe96q7UaHaq7ktX3qV6PQNLPRCcYEOsHkfvcS7zFo6yC2h25nuN9wOlfsnGmfxKgowr+czs3ly7EuUYLiY0aTr16qq9LuRMDWibB/EWKbn+By7zI6rBb7QqMpYGdFpxol6VmrNM4O+n4aTUtNJxgT6ATzdIhPjGfwtsFsvriZwb6D6VG5R+adgLv79hE2ajRxISEUaNWKYsOHYeXk9GCj8OOw4TM4/TcUdueM9ydMv1Cev45dwUIpXvV2YXDz8jrRaFoKOsGYQCeYp0d8UjzDdwzH/5w/H/p8SB/PPib1S4qL4/rcH7j+/fcoe3uKfTIEh3bt/ruk+b5TfxsSTcRxKFWbKzVHMveUA0v3nsdCQd96ZXinnrueOtM0dIIxiU4wT5eEpARG7hrJnyF/0s+rH/28+j2cKNJx78wZwkaPJmbffuz9/Cg+dgy2bm4PNkpMgIM/GhY4u3sNvDpx2Wcgk3bf5Y9Dlyma35bBzSvQvqorlvpiAO05phOMCXSCefokJiUyZvcYVp9eTa8qvRhQdYDJSUaSkri5ciXhU6ch9+7h1O9dHHv1QtnYPNgw9hbsnA67v4HEe+BSjVDnZowPKYv/5TxUci7AZ60rUquMU9ofpGnPOJ1gTKATzNMpSZKYsGcCy08up1ulbgzxHWJykgFIiIjgysSJ3P5rPTZly+A8bjz2VX0ebnjzIhxZAcG/Q1gQAFEOFfk12odfo31wr1iV4S1fxL1IvuzaNU17KugEYwKdYJ5eIsKUwCksO76MNyu8yfAaw7FQj7b6xO0tW7gybjwJYWEU7PQmRQcNwjJ//rQb3zgPx/+A42vg4l4AzogLfyX5YV3lNd5o3ZKCeW0fd7c07amgE4wJdIJ5uokIX+3/ioXHFtK+XHtG1Rz1yEkmKTqaiFmziFyyFCtHR4qN/Iz8TZtmfER0KwxO/EnckdVYXfwHC5K4SDFulGpBxUZdsS5ZHR7hiErLASJw9RiEbIULu6GEN9R4F2zT+QVDe4BOMCbQCebpJyJ8HfQ1cw/PpU2ZNoyrNQ5Li0cvZhlz5AhhI0dx78QJ8jVuTPGRn2FdvHjmHaOvERawivC9y6kYcxAblUhsnuLYeryKqvQqlHwJshCPZgZRlwwJJWQLhGyD6HDD+w4vQNRFsHeEuh+Dby+w1nXqMqITjAl0gnl2fHfoO+YEzaGlW0sm1pmIlcWjX04s8fFE/vgjEbO/RllaUmTgQAp1ehNlmXmCEBF2HjnDznVLqBa9gwaWh7EhHvIWhRdbQaU2ULouWFpnZfe0rIi9Bed2GhPKVrh20vB+3iLg3gDcG4J7fXBwhdB9sHm8oV0BF6j/CXh30X9f6dAJxgQ6wTxb5h+Zz4wDM2haqilT6k7BOov/OcRdvMiVMWOJ3rUL2/LlKdS1Cw6tWz9YciYd8YlJ/BJwge82HMInLpA+jkfxiNmLRfxdyFMIKrwMFdtAmYZgpc/ZZKvEeEOiuJ9QQveBJIK1PZSqZUwoDaBY5fSnMEO2GRJNaCAUcoOGI6BKe7Aw5+ryTx+dYEygE8yz58djPzJ131QauDbgywZfYmNpk3mnNIgIt9au4/q8edw7cQKL/Pkp2K4thTp1wqZ06Uz7R8XEM2fLaRbuOks+ywTGV75KC8sArE75w70osMkPFVoYkk3ZJmCTefLSUhGBiH//m/Y6txPi7oCygBI+/yWUF/weLZmLwEl/Q6K5ehSKVoZGn0GFlvrcmpFOMCbQCebZ9MuJX5iwdwK1XWozo8EM7KyyPp8uIsQcOMCNZT9xa8MGSEggb506FOrcmXz162U6fXb+ejST/zrBX0evULyAHUObufFq/tNYnFgDJ9ZCTKThN+yyTaDSq1CuGdjpZZ7TdfuK4Sjj/lHK7TDD+4Xd/0sobnUNR4uPKynJUIF7y0SIPAMu1aDxKMNnPOd0gjGBTjDPrlUnVzF291j8nP2Y1XAW9taPf4QQHx7OzRUruPnrchLCw7F2caHgm29QsEMHrApl/B9awNlIPl8bzOHQKDxdHfisVSX8ShaA87sMlz4f/wPuXAVLG/7f3p3HR1Xf+x9/fSbrJJAQ9jXsIKsKbqgoqIigorTKphVZ6rW13t7rtbbW2l9vbW9ra3+2914fWhAEl4BWK1JRkUXEBUWqoKyyCwoCScg+me1z/zgTmIQAkzAnC/k8H488mJmzzOfrmHnne873fA89r3J6Nn3HQFrLM667USsvhr0fHsExnPkAAB54SURBVA+UQ5ud170tI+dRIj9ZXd2rIRSEDTmw6lEo3A/dr4CrfgldLnTvPRs4C5gYWMCc3RbvXMzDHzzM+W3P54mrnyA9KT0u+9VAgKIVK8nPyaF07VokOZmMsWPJum0K3kGDTrpdOKwsWv81f3hrGwcLfYwZ2J4Hx/Qju1Wa89fy/rWwebETOAX7wJPoDAzoPw7OuQGatY1L/Q1aKAjffHY8UPathXAAElKg67DjvZT2g+v+vEjAB/98BlY/5kwl1GeMc+is/cC6raMBsICJgQXM2e/N3W/y4HsPMrD1QJ685kmaJ8f3Oofy7dvJX7CAgkWvES4tJXXQILKmTCFj7Bg8KdUf9y/zh5j93i6eXLWTUFi587Ju3DOyF5neyKAEVfjm0+Nhk7cLEOdEdb9x0O9GyOxU7b4bHVXI3Xk8UHa/55yjQqDD4OOBkn0JJDWQGa3Li+Hjp+CD/3ZqHfhdGPFzaN2rviurMxYwMbCAaRqW7V3GA+8+wDktz+GpUU+RmRL/WySHiospWPQa+Tk5+HftIqFFC1rcegstJk4iuXP1YfBtoY/Hlm7j5U/308KbxL+P6sOUi7JJTIj6y7zigsAti53AObzFeb3TBU7Ppt84aNm92v03WMWHYXfkPMrOVc5hJ4AW2VHnUa6E9Fb1WGQMyvLhw/+Bj56EYDmcfxtc+VNn2PNZzgImBhYwTceqfau4b9V99GzRk1mjZpGVGoeTwNVQVUo//pj8F16gaMVKUKXZiBFk3XYb6ZcOQ6o5rLPx6wJ+s2QzH+3Ko1fbZjw0th8j+rapfjaBI9ududG2LIYDG5zX2g+Cfjc5gdOmryvtOiP+UvjqQ6eHsnMVfPuF83pqphMkPUY4w7azujfOUVrFh+C9P8G6uc7zC2Y4F2w2a1O/dbnIAiYGFjBNy/tfv8+/vfNvdGnehdnXzqa1193ZkAMHDpD/4oscfelvhPLySO7alawpk8kcP56EjMojxVSVZZu/5XdvbmX3kRKG927NQ9f345z2pxhRlr/HGRywebFz/gagzTlOr6b/OGg3sH6+sMMhJ/x2vQM733Hmbgv5nQEMXS4+fpFjx/POrlkOju6Ddx+F9S9Aohcu+QFcei94W9R3ZXFnARMDC5im56MDH3Hvinvp0KwDT1/7NG3T3D9xHvb7KVq6lPwXcihbvx7xesm84QaybptC6jnnVFrXHwzz/Ed7+cuK7RT5Aky8MJv7RvWhTfPTXMdR+A1sed3p2ez9ADTs9Aj6j3N6N52GuBs2ebujzqOsdg4fgRNyPUY4gdJ1GCTHZ6BFg3ZkuzO0edPfnV7aZT925jk7i9puARMDC5imad3Bddyz4h5ae1szZ/Qc2qfHMOdYnPg2byYvJ4fC15egPh/eoUPJmjKZjFGjKt2X5mipn/9esYNn1+whJdHDD0f2Ysbl3UlNiuEv/uLDsG2J07PZ/S6Eg5DR2Rkc0H+c04s4055DaV7kPMoqp5dydK/zekan4+dRelzZNEa+ncyBz+Gd38KXbzlTBl1xPwy986yYwaFBBIyIXAf8BUgAnlbV31dZngI8CwwFcoGJqronsmww8FcgAwgDF6qqL2rbxUAPVR0Y9dq9wI+AILBEVR84VX0WME3X+kPr+cHyH5CZksmc0XPo1KxuR2WFjh7l6N9fJX/BAgL79pHQpjVZt06gxcQJJLVrd2y9XYeL+d2bW1m2+Vs6tfDy0zHncOPgDrHf/6YsH7a95fRsdqxwbqCW3hb63eAcSut2eWzzbQV8sO+j44FyYAOgkJLhDKXuMcI5j9KqV+M8j+Kmrz6GFb+Gve87E2te+VM4dzIkNN7bb9d7wIhIAvAlMArYD3wCTFbVzVHr/BAYrKp3i8gkYLyqThSRROBT4HuqukFEWgFHVTUU2e47wC2RbQdGXhsJPARcr6rlItJWVQ+dqkYLmKZt45GN3LXsLtKT0plz7RyyM7LrvAYNhyl5/33yX8ihePVq8Hhofs01ZE2ZQtpFFx4Lkg93HOE3S7aw+UAh52e34OEb+jMku4YDFcqLnClQtiyG7cvg2Pxokck4e4w4/td1OOycjK8IlK/WQNDnXJvT+aLjgdJxSKP+oqwzqs4hxBWPOEPQW/WCkT+H/uMb5TxnDSFghgG/UtXRkecPAqjq76LWWRpZZ00kVA4CbYAxwBRVvb2a/TYD3gLuAl6KCpiXgFmqujzWGi1gzNa8rXz/7e+T7Elm9ujZ9MjsUW+1+PftI3/BQo6+8grhggJSeveixeTJZI67iYRm6YTCyiuf7uePS7dxuKicG8/tyAOj+9KlZS1mKfCXws4VzmG0L9+C8kKnN9JntHOSfve7UJrrrNum3/FA6Xqp3TPlTKg6UwSt/I0z5LzdILj6YWeKoEbU82sIAXMLcJ2qzow8/x5wsar+KGqdjZF19kee7wQuBm7HOWzWFidwFqrqHyLrPA6sBj4DXo8KmPXAa8B1gA+4X1U/qaauu3DCiezs7KF79+51ofWmMdmev52Zb89EEGZfO5veWb3rtZ6wz0fhkjfIf+EFfJs340lPJ/Pmm8maMpmUnj0pKQ/y19W7mLV6J2GFGZd354cjetI8tZZTywfLnfm9trwGW99wRnv1GHH8J6NDvJpWrwKhACWBEkqCJZQESigNlFISKKEsWMag1oNol97u9DuJl3AIvngZVv2XMxqw80XOPGfdh9ddDWegIQTMrcDoKgFzkareG7XOpsg60QFzETANuAe4ECgFVgC/wDlP84iq3igi3agcMBuBlcCPI9u9iHOO5qQNtB6MqbCrYBczl84kGA4y+9rZ9G1Z/9eTqCq+DRvIy8mh6M230ECAtEsuIeu2KTQfOZIDxQEeW7qNv3/2Na2bJXPfqL5MuKBz5Qs1a/6mzr8N4K/pYDhYKQiqBkNJoITSYOkJzyseH1s36PwbCAdO+l6JnkRu6HED0wZMo0eLOuzFhgLw2XPw7h+h6BtngMTVDzsTazZgDSFgzuQQ2UScns2dkfUexumVFAMPA34gEaeH86GqjhCRt4Dfq+qqyDY7gUtU9fDJarSAMdH2Fu5lxtIZlAXLmDVqFgNaD6jvko4J5uZy9OVXyF+4kOCBAyR26EDWxAm0uPVWNpcl8Mjrm/lkTz592zXnoev7cUWfur/ILxQOHfuCr0koVARApVAIlOAP+2N63wRJIC0pjfSkdNIT00lPSseb5D32+NiyyE9aYlql1xMkgdd3vc6r21/FF/IxsstIpg+cznltz3P5v1iUQBl8Mse5YLMsz5l7buRD0K5/3dVQAw0hYBJxTvJfDXyNc5J/iqpuilrnHmBQ1En+76jqBBHJwum1XI4TJm8Bj6vqkqhtu1G5B3M30FFVfykifSLbZ1sPxtTE/qL9zHx7JgXlBTw16inObXNufZdUiQaDFK9aRX5ODiUfroGkJDJGjyZryhTeSWjD75duYV9+McP7tOS+Ub3o2tpLKBwipCGC4SAhDREKhwhqkFA4RFjDxx5XrOML+ip96Z8QGtX0FkoDpfhCvtM3ABCk8hd/ovO4akhEB0NaUtqx9aK3SU9KJyUhJfZRdaeQ58tjwdYFLNi6gILyAoa0HcKMQTMY3ml4XPYfE1+hM/XMmv91BmUMngAjfubcjqABqfeAiRQxFvgzzjDluar6WxH5NbBOVReLSCrwHHA+kAdMUtVdkW1vBx4EFHij6pDjagImGZgLnIcTSver6spT1WcBY6pzoPgAM96eQW5ZLvcNvQ9vkrfSl3LFF3FYwyf94o5+PazhSuvU5Mu+6jbR79nmsJ+Rn5Rz+ecB0sphVztYOtTDB/0Ff1J8vxAr/uqv+LKv2gs4WW/Bm+g9offgTfTW3Rd2LZQGSnll+ys8u/lZDpYcpHdWb6YNmMZ13a8jyVNHt1AuzYMP/gwfz3Jmkz7/e85tnDM61s37n0aDCJiGzgLGnMyh0kPMfHsmuwt2x7xNgiQ4P54EEiWRBE/l5x7xkOhJPPZagiRUel5pm6h1qi6ruk1SeYhOH+ygy7JNNNuXSyA9lQMjBrK4dzeWFyrJCYlcdU57rujdjuSExJO+Z6LHqTE1IbVSUHgTvXik8Q2lrVBcHuRggY9vC30cLPBxsPD4428LfXxbWE5IleQEDylJHuffRA9JiUpp0jqOJCylTL4mhVZ0TxpLj9SRpCWlkZzoISUxgZREZxvnufNvxbLkKstOtjwpQaoP3cID8N5j8M/5zsWxF86Ey++r94lALWBiYAFjTsUf8vN18dfVhkX084pAqO+/ylWVsnXryHshh6LlyyEUQi4axt+6XMzcQAc6t0rnwTH9GDOwfb3XGg+hsJJbXM7BqLBwHpdHPfZRXB48YduM1ETaZ6bSLsP5SUoQyoNh/MHwsX/9wTD+UBhfMECRfEFRyjKCybsglA6Fl1Kedyl+f3xuISBC5ZBK8JCSlHDstc5yiEmlOQwvXYFfUliZdSsftJlEOCXjWGhFB12lADu2Pw8pUcs7Z6XRMr12txS3gImBBYw5WwW+PcTRl14i/6UXCR0+Qqh9R17vNoznW55Lvz6d+MX1/Tm3S8OdhLHMH6omOCo/PlRUTihc+fsrwSO0bZ5C+8xU2kfCo7rH3uTaTZWz/tB65mycw6p9q/AmehnfazyT+95Oq9T2lcMpVBFWoUqBVd3yY8tCYcoDx5dVWh5Z1q58D5NKnmN44EMKaMZ8z3ieD19LYSiJ8mCYmnydP3LzQL53Se3uBGoBEwMLGHO2U7+fouXLycvJoWzdPwknJfNe9vm83GUYg0ZezE9G96Vji7q7kVc4rOSV+iuFxbeRw1YHog5jFfpO7HU0T0mkXaWwSDkhOFo1SyHB437vbOfRnczdOJc3dr2BoozpPoZpA6fRJ6uP6+8NOHf9XPkb2LEcmrWDK36CDrmDoCRVE2ZOyFXqmQXD9G3fvHYX6WIBExMLGNOU+LZtIz9nAQWvvYb6fGxt2ZW3el1Gv4nj+f4155CecmZTvvgCIQ4VRg5ZRQXHwYLoXoePQKjyd45HoE3zymHRLsMJjGOPM1NJlzBhn49wWRlaVuY8Li1DfWWEy8oIl/kIl5WiZb7IepHHZZF1Sp1ttCyyfsU65X7Shgyh1cwZeM+t2ajBgyUHeXbzs7z85cuUBcsY3mk40wdOZ2i7oXVzGHLPB7DyEWcqnxbZMOJBGDzR9dsgWMDEwALGNEWhwkIKFi3i0PM56Fd7OZqczuo+l9JnxlTGjR5yQg9AVTlaGqgUHAeqHq4qKKWkqJTUkJ+UYIDUkJ/UkJ9MCdE+BdokKa2TwrRMUFpIkExPiGYaIE2DpIYC4KsIiLJKARD9mOCJvZpTEsHj9SJeLx6vF483FfGm4UlNPf56aip4PBQtX064sJC0Cy+k1cwZpF9xRY0CoqC8gIVbF5KzNYc8Xx6D2wxm+sDpjOwy0v0BEqrORKYr/hMOfg6t+8JVDzmTmboUchYwMbCAMU2ZhsOUrFnD7qfn4/nofVRhc/dzkX4D8BWV4C8uJVhaSrisjKRAOSmhACkhP6lBPymhAGnhAN6wn+Sgn6Tgya+SPxlJTj7+5Z+aiqR58aRWeez1It5UPN40PN5IMKRWhEVknTQvklplHa/X2X+MX7Ch4hIKXnmZ3HnzCR44QErv3rScMZ3MsWMr3UbhdMqCZby24zXmbZrH18Vf0z2zO9MGTOOGHjeQFMus1WciHHYmMn3nt3DkS+hwHlz1MPS6Ou5BYwETAwsYYxz+/ftZ9z9zSVz6D5r7igkjBJKSCSWnoCmRL/A0L0lpaaQ0TyO1WToJac4XuqRG9xCq/9KvFCTeNDypKUhiw5uFWQMBCt94g9yn51C+fTuJ7dvTcupUWtx6KwnNYr9hWDAcZNneZcz5Yg7b8rfRNq0td/S/g1v63EJ6kss3HguH4PMXYdXv4OhXkH2pM/1M10vj9hYWMDGwgDGmsnAwiAaDeFLic3V8Y6WqlLz3HrlPz6F07Vo8GRlkTZ5My9tvI7FN7NPwqCoffvMhczfOZe3BtTRPbs6kvpOY0m+K67fsJuiHT+fD6j9C8bfQ6xq46hfQ8fwz3rUFTAwsYIwxp1P2+efkPj2HomXLkKQkMm++mZbT7iSle/ca7WfjkY3M3TiX5XuXk+RJ4uZeN3PngDvpktHFpcoj/KWwdpYzM0BZPvS/yZnnrE3tJ3S1gImBBYwxJlb+PXvIfWYeBa++igYCNL/mmlqNPNtTsId5m+axeOdiQhri2q7XMn3gdPq16udS5RG+AljzhPMTKIXr/wQXTK/VrixgYmABY4ypqeCRI+Q9/zz5OQvOaOTZ4dLDPLflOf627W8UB4oZ1mEY0wdN5+L2F7t7eLLkCLz/uBMurXrWahcWMDGwgDHG1Fa8Rp4V+Yt4adtLPL/leY6UHaF/q/5MHzida7KvIcHl61lqywImBhYwxpgzFa+RZ+Whcv6x8x/M2zSPvYV7yW6ezdQBU7mp102kJKS42IKas4CJgQWMMSZe4jXyLBQOsXLfSuZ+MZeNuRtpldqK2/vfzoS+E8hIznCxBbGzgImBBYwxxg1ln39O7py5FL39dq1Hnqkqaw+uZe7GuXz4zYekJ6Uzoc8Ebu9/O23T2rpY/elZwMTAAsYY46Z4jTzbkruFZzY+w9K9S0mQBG7seSN3DriT7pk1GyodLxYwMbCAMcbUhXiNPNtXtI/5m+azaMci/CE/V2VfxfSB0xncZrCL1Z/IAiYGFjDGmLoUr5FnuWW55GzNYeHWhRT6C7mw/YVMHzidyzpeViczMFjAxMACxhhTH+I18qwkUMIrX77C/M3zOVR6iL5ZfZk2cBqju40m0ePeXG8WMDGwgDHG1Kd4jTwLhAIs2b2EZzY+w66CXXRq1ok7+t/B+N7j8SbG/4ZyFjAxsIAxxjQU8Rh5FtYw7+57l7kb57L+8HqyUrKY3G8yk/tOpkVq/G6RbQETAwsYY0xD49+zh9x58yj4+5mNPPv020+Zs3EOq/evxpvo5bu9v8vUAVNpn97+jGu0gImBBYwxpqGK18iz7fnbeWbjM7y5+00AxvYYy7QB0+iV1avWtVnAxMACxhjT0MVr5NmB4gM8u/lZXtn+CmXBMn5+8c+ZfM7kWtVkARMDCxhjTGOhgQCFb77pjDz78stajzw76jvKgm0LuL779WRnZNeqFguYGFjAGGMam3iNPDsTsQaMx+UirhORbSKyQ0R+Vs3yFBF5MbL8YxHpFrVssIisEZFNIvKFiKRW2XaxiGysZp/3i4iKiMv3IzXGmLonIjS74gq6Pjufbi+9SPqwYeTOmsWOq6/hwC//H+W7d9d3ice4FjAikgA8AYwB+gOTRaR/ldVmAPmq2gt4HHg0sm0i8Dxwt6oOAEYAgah9fwcoruY9uwCjgK/i3R5jjGlovIMH0/kvf6bnm2+Q+Z3xFCxaxK6x17P/3n+lbMOG+i7P1R7MRcAOVd2lqn5gIXBTlXVuAuZHHr8MXC3O8Ihrgc9VdQOAquaqaghARJoB9wG/qeY9HwceAJrucT9jTJOT3K0bHX71K3qtXEGru/+FkrVr2TNxEnu/dwfF775LfZ0KcTNgOgH7op7vj7xW7TqqGgQKgFZAH0BFZKmIfCoiD0Rt8wjwJ6A0ekciMg74uiKUjDGmqUls3Zq2P/4xvVeuoN2DP8O/fz/7/uVudo+7iaOLFqF+f53W42bAVDdQu2qMnmydROBy4LbIv+NF5GoROQ/opaqvVtqJSBrwEPDL0xYlcpeIrBORdYcPH46hGcYY07h40tNpOXUqvd5eSsc/PAoiHPjZg+y4djS5z8wjVFxSN3W4uO/9QJeo552Bb062TuS8SyaQF3n9XVU9oqqlwBvAEGAYMFRE9gDvA31EZBXQE+gObIgs6wx8KiInXLKqqrNU9QJVvaBNHY24MMaY+iBJSWSOG0f31xbRZfYskrOzOfToo+y46ioKXl/i+vu7GTCfAL1FpLuIJAOTgMVV1lkMTI08vgVYqc7BwqXAYBFJiwTPlcBmVX1SVTuqajecns2XqjpCVb9Q1baq2i2ybD8wRFUPutg+Y4xpFESEZsOHVxp5ltyls+vv69p8zqoaFJEf4YRFAjBXVTeJyK+Bdaq6GJgDPCciO3B6LpMi2+aLyP/HCSkF3lBV9+PWGGPOchUjz+qCXWhpF1oaY0yNNIgLLY0xxjRdFjDGGGNcYQFjjDHGFRYwxhhjXGEBY4wxxhUWMMYYY1xhAWOMMcYVTfo6GBE5DOyt5eatgSNxLKcxsDY3DdbmpuFM2txVVU8711aTDpgzISLrYrnQ6GxibW4arM1NQ1202Q6RGWOMcYUFjDHGGFdYwNTerPouoB5Ym5sGa3PT4Hqb7RyMMcYYV1gPxhhjjCssYE5DRK4TkW0iskNEflbN8rtF5AsRWS8i74tI//qoM55O1+ao9W4RERWRRj/6JobP+U4RORz5nNeLyMz6qDOeYvmcRWSCiGwWkU0iklPXNcZTDJ/x41Gf75cicrQ+6oynGNqcLSLviMhnIvK5iIyNawGqaj8n+cG5UdpOoAeQDGwA+ldZJyPq8Tjgrfqu2+02R9ZrDqwGPgIuqO+66+BzvhP43/qutY7b3Bv4DMiKPG9b33W72d4q69+Lc5PEeq/d5c94FvCDyOP+wJ541mA9mFO7CNihqrtU1Q8sBG6KXkFVC6OepuPcgbMxO22bIx4B/gD46rI4l8Ta5rNJLG3+PvCEquYDqOqhOq4xnmr6GU8GFtRJZe6Jpc0KZEQeZwLfxLMAC5hT6wTsi3q+P/JaJSJyj4jsxPnC/dc6qs0tp22ziJwPdFHV1+uyMBfF9DkD340cRnhZRLrUTWmuiaXNfYA+IvKBiHwkItfVWXXxF+tnjIh0BboDK+ugLjfF0uZfAbeLyH7gDZyeW9xYwJyaVPPaCT0UVX1CVXsCPwV+4XpV7jplm0XEAzwO/EedVeS+WD7nfwDdVHUwsByY73pV7oqlzYk4h8lG4PxF/7SItHC5LrfE9LscMQl4WVVDLtZTF2Jp82Rgnqp2BsYCz0V+x+PCAubU9gPRf6l25tRdyIXAza5W5L7Ttbk5MBBYJSJ7gEuAxY38RP9pP2dVzVXV8sjT2cDQOqrNLbH8v70feE1VA6q6G9iGEziNUU1+lyfR+A+PQWxtngG8BKCqa4BUnDnK4sIC5tQ+AXqLSHcRScb5H29x9AoiEv0Ldz2wvQ7rc8Mp26yqBaraWlW7qWo3nJP841R1Xf2UGxexfM4dop6OA7bUYX1uOG2bgUXASAARaY1zyGxXnVYZP7G0FxHpC2QBa+q4PjfE0uavgKsBRKQfTsAcjlcBifHa0dlIVYMi8iNgKc6IjLmquklEfg2sU9XFwI9E5BogAOQDU+uv4jMXY5vPKjG2+V9FZBwQBPJwRpU1WjG2eSlwrYhsBkLAT1Q1t/6qrr0a/H89GViokWFVjVmMbf4PYLaI/DvO4bM749l2u5LfGGOMK+wQmTHGGFdYwBhjjHGFBYwxxhhXWMAYY4xxhQWMMcYYV1jAGOMSEfmViNzfAOrYE7mOxZg6ZQFjjDHGFRYwxtSAiKSLyBIR2SAiG0VkYnQPQUQuEJFVUZucKyIrRWS7iHw/sk4HEVkdue/IRhEZHnn9SRFZF7n3yn9GveceEfkvEVkTWT5ERJaKyE4RuTuyzojIPl+N3L/lqermlBKR20VkbeS9/yoiCW7+9zJNmwWMMTVzHfCNqp6rqgOBt06z/mCcKYSGAb8UkY7AFGCpqp4HnAusj6z7kKpeENnmShEZHLWffao6DHgPmAfcgjMP3K+j1rkI58rsQUBP4DvRhUSmApkIXBZ57xBwWw3abkyN2FQxxtTMF8BjIvIo8LqqvidS3aS1x7ymqmVAmYi8gxMCnwBzRSQJWKSqFQEzQUTuwvm97IBzA6jPI8sqpjL5AmimqkVAkYj4omY4XququwBEZAFwOfByVC1X40zS+UmkZi/QmO/xYho4CxhjakBVvxSRoThTm/9ORN7GmZ+s4mhAatVNTtyFrhaRK3B6Ns+JyB9xeib3Axeqar6IzKuyr4qZnMNRjyueV/wen/BeVZ4LMF9VHzxNM42JCztEZkwNRA5xlarq88BjwBBgD8en7/9ulU1uEpFUEWmFc1+VTyI3tDqkqrOBOZF9ZAAlQIGItAPG1KK8iyIz53pwDoW9X2X5CuAWEWkbaUvLSC3GuMJ6MMbUzCDgjyISxplB+wc4h5rmiMjPgY+rrL8WWAJkA4+o6jciMhX4iYgEgGLgDlXdLSKfAZtwpsT/oBa1rQF+H6lxNfBq9EJV3SwivwDejoRQALgH2FuL9zLmtGw2ZWPOAiIyArhfVW+o71qMqWCHyIwxxrjCejDGGGNcYT0YY4wxrrCAMcYY4woLGGOMMa6wgDHGGOMKCxhjjDGusIAxxhjjiv8DU2dQ04RiFogAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3.best_score_, gsearch3.best_params_))\n",
    "test_means = gsearch3.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'reg_alpha': 1.5, 'reg_lambda': 0.5}, -0.06553447916454043)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#找出最佳 alpha & lambda\n",
    "\n",
    "reg_alpha = [ 1.5, 2]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test5_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "\n",
    "\n",
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=98,  #第一轮参数调整得到的n_estimators最优值  98\n",
    "        max_depth=3,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.8,\n",
    "        objective= 'binary:logistic',\n",
    "        seed=3)\n",
    "\n",
    "gsearch5_1 = GridSearchCV(xgb5, param_grid = param_test5_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch5_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch5_1.best_params_,     gsearch5_1.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#以上为参数调试过程，可以根据实际情况通过调整补长以及区间来不断的优化性能， 因为时间关系就不再进一步处理。\n",
    "\n",
    "最终模型参数如下：\n",
    "\n",
    "#reg_alpha': 1.5, 'reg_lambda': 0.5\n",
    "\n",
    "#colsample_bytree': 0.9, 'subsample': 0.8\n",
    "\n",
    "#learning_rate =0.1\n",
    "\n",
    "#n_estimators=98\n",
    "\n",
    "#max_depth=3\n",
    "\n",
    "#min_child_weight=3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.8,\n",
       "       colsample_bynode=1, colsample_bytree=0.9, gamma=0,\n",
       "       learning_rate=0.1, max_delta_step=0, max_depth=3,\n",
       "       min_child_weight=3, missing=None, n_estimators=98, n_jobs=1,\n",
       "       nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=1.5, reg_lambda=0.5, scale_pos_weight=1, seed=3,\n",
       "       silent=None, subsample=0.3, verbosity=1)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb5 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=98,  #第一轮参数调整得到的n_estimators最优值  98\n",
    "        max_depth=3,\n",
    "        min_child_weight=3,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.9,\n",
    "        colsample_bylevel = 0.8,\n",
    "        objective= 'binary:logistic',\n",
    "        reg_alpha=1.5,\n",
    "        reg_lambda=0.5,\n",
    "        seed=3)\n",
    "\n",
    "xgb5.fit(X_train,y_train,eval_metric='auc')\n",
    "\n",
    "#设置验证集合 verbose=False不打印过程\n",
    "xgb5.fit(X_train, y_train,eval_set=[(X_train, y_train)],eval_metric='auc',verbose=False)\n",
    "\n",
    "#获取验证集合结果\n",
    "#evals_result = xgb5.evals_result()\n",
    "#y_true, y_pred = y_test, xgb5.predict(X_test)\n",
    "#print\"Accuracy : %.4g\" % metrics.accuracy_score(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>columns</th>\n",
       "      <th>importance</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>Monthly_Income</td>\n",
       "      <td>0.068669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Source_S122</td>\n",
       "      <td>0.057681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>Var5</td>\n",
       "      <td>0.057675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Gender_Female</td>\n",
       "      <td>0.051990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Gender_Male</td>\n",
       "      <td>0.034729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Filled_Form_Y</td>\n",
       "      <td>0.033912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Var1_HAXC</td>\n",
       "      <td>0.033861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>Var4</td>\n",
       "      <td>0.031429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Source_S137</td>\n",
       "      <td>0.030346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>Loan_Amount_Applied</td>\n",
       "      <td>0.030065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mobile_Verified_N</td>\n",
       "      <td>0.027219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Var1_HAXB</td>\n",
       "      <td>0.026599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>Existing_EMI</td>\n",
       "      <td>0.025425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Device_Type_Mobile</td>\n",
       "      <td>0.025368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>DOB_Year</td>\n",
       "      <td>0.025263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Source_S133</td>\n",
       "      <td>0.024902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>Processing_Fee</td>\n",
       "      <td>0.024240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>Loan_Amount_Submitted</td>\n",
       "      <td>0.024049</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Source_S143</td>\n",
       "      <td>0.023688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>EMI_Loan_Submitted</td>\n",
       "      <td>0.023560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Filled_Form_N</td>\n",
       "      <td>0.023128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>Lead_Creation_Day</td>\n",
       "      <td>0.023065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>Interest_Rate</td>\n",
       "      <td>0.022734</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>Lead_Creation_Month</td>\n",
       "      <td>0.022678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Var1_HBXD</td>\n",
       "      <td>0.022313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Var1_HAXM</td>\n",
       "      <td>0.022158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>Loan_Tenure_Applied</td>\n",
       "      <td>0.021061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Source_S134</td>\n",
       "      <td>0.020654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Var2_G</td>\n",
       "      <td>0.019376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>DOB_Month</td>\n",
       "      <td>0.019188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Source_S151</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Source_S153</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Source_S136</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Source_S150</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Var2_D</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Var1_HAYT</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Var1_HAZD</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Var1_HBXC</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Var1_HBXH</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Var1_HCXD</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Var1_HCXF</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Var1_HCXG</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Var1_HCYS</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Var1_HVYS</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Device_Type_Web-browser</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Var2_A</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Var2_F</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Source_S144</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Source_S123</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Source_S124</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Source_S125</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Source_S127</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Source_S129</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Var1_HAVC</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Source_S135</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Source_S138</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Source_S139</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Source_S140</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Source_S141</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Source_S130</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>81 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    columns  importance\n",
       "64           Monthly_Income    0.068669\n",
       "34              Source_S122    0.057681\n",
       "74                     Var5    0.057675\n",
       "0             Gender_Female    0.051990\n",
       "1               Gender_Male    0.034729\n",
       "24            Filled_Form_Y    0.033912\n",
       "7                 Var1_HAXC    0.033861\n",
       "80                     Var4    0.031429\n",
       "45              Source_S137    0.030346\n",
       "71      Loan_Amount_Applied    0.030065\n",
       "2         Mobile_Verified_N    0.027219\n",
       "6                 Var1_HAXB    0.026599\n",
       "73             Existing_EMI    0.025425\n",
       "25       Device_Type_Mobile    0.025368\n",
       "65                 DOB_Year    0.025263\n",
       "41              Source_S133    0.024902\n",
       "78           Processing_Fee    0.024240\n",
       "75    Loan_Amount_Submitted    0.024049\n",
       "50              Source_S143    0.023688\n",
       "79       EMI_Loan_Submitted    0.023560\n",
       "23            Filled_Form_N    0.023128\n",
       "70        Lead_Creation_Day    0.023065\n",
       "77            Interest_Rate    0.022734\n",
       "69      Lead_Creation_Month    0.022678\n",
       "15                Var1_HBXD    0.022313\n",
       "9                 Var1_HAXM    0.022158\n",
       "72      Loan_Tenure_Applied    0.021061\n",
       "42              Source_S134    0.020654\n",
       "33                   Var2_G    0.019376\n",
       "66                DOB_Month    0.019188\n",
       "..                      ...         ...\n",
       "53              Source_S151    0.000000\n",
       "54              Source_S153    0.000000\n",
       "44              Source_S136    0.000000\n",
       "52              Source_S150    0.000000\n",
       "30                   Var2_D    0.000000\n",
       "10                Var1_HAYT    0.000000\n",
       "11                Var1_HAZD    0.000000\n",
       "14                Var1_HBXC    0.000000\n",
       "16                Var1_HBXH    0.000000\n",
       "18                Var1_HCXD    0.000000\n",
       "19                Var1_HCXF    0.000000\n",
       "20                Var1_HCXG    0.000000\n",
       "21                Var1_HCYS    0.000000\n",
       "22                Var1_HVYS    0.000000\n",
       "26  Device_Type_Web-browser    0.000000\n",
       "27                   Var2_A    0.000000\n",
       "32                   Var2_F    0.000000\n",
       "51              Source_S144    0.000000\n",
       "35              Source_S123    0.000000\n",
       "36              Source_S124    0.000000\n",
       "37              Source_S125    0.000000\n",
       "38              Source_S127    0.000000\n",
       "39              Source_S129    0.000000\n",
       "4                 Var1_HAVC    0.000000\n",
       "43              Source_S135    0.000000\n",
       "46              Source_S138    0.000000\n",
       "47              Source_S139    0.000000\n",
       "48              Source_S140    0.000000\n",
       "49              Source_S141    0.000000\n",
       "40              Source_S130    0.000000\n",
       "\n",
       "[81 rows x 2 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"columns\":list(X_train.columns), \"importance\":list(xgb5.feature_importances_)})\n",
    "df.sort_values(by=['importance'],ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
